Merge pull request #1842 from Nordix/fix-ironic-bmo-upgrade-1.6/adil

🐛 Fixing bmo and ironic version for clusterctl tests

scripts/ci-e2e.sh

@@ -44,8 +44,8 @@ export IMAGE_OS=${IMAGE_OS}
 export FORCE_REPO_UPDATE="false"
 EOF
 if [[ ${GINKGO_FOCUS:-} == "features" ]]; then
-    mkdir -p "$CAPI_CONFIG_FOLDER"
-    echo "enableBMHNameBasedPreallocation: true" >"$CAPI_CONFIG_FOLDER/clusterctl.yaml"
+  mkdir -p "$CAPI_CONFIG_FOLDER"
+  echo "enableBMHNameBasedPreallocation: true" >"$CAPI_CONFIG_FOLDER/clusterctl.yaml"
 fi
 # Run make devenv to boot the source cluster
 pushd "${M3_DEV_ENV_PATH}" || exit 1
@@ -124,5 +124,68 @@ export CONTRACT_TO="v1beta1"
 # image for live iso testing
 export LIVE_ISO_IMAGE="https://artifactory.nordix.org/artifactory/metal3/images/iso/minimal_linux_live-v2.iso"
 
+# Generate credentials
+BMO_OVERLAYS=(
+  "${REPO_ROOT}/test/e2e/data/bmo-deployment/overlays/release-0.5"
+)
+IRONIC_OVERLAYS=(
+  "${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-23.1"
+  "${REPO_ROOT}/test/e2e/data/ironic-deployment/overlays/release-24.0"
+)
+
+# Create usernames and passwords and other files related to ironi basic auth if they
+# are missing
+if [[ "${IRONIC_BASIC_AUTH}" == "true" ]]; then
+  IRONIC_AUTH_DIR="${IRONIC_AUTH_DIR:-${IRONIC_DATA_DIR}/auth}"
+  mkdir -p "${IRONIC_AUTH_DIR}"
+
+  # If usernames and passwords are unset, read them from file or generate them
+  if [[ -z "${IRONIC_USERNAME:-}" ]]; then
+    if [[ ! -f "${IRONIC_AUTH_DIR}/ironic-username" ]]; then
+      IRONIC_USERNAME="$(uuid-gen)"
+      echo "${IRONIC_USERNAME}" > "${IRONIC_AUTH_DIR}/ironic-username"
+    else
+      IRONIC_USERNAME="$(cat "${IRONIC_AUTH_DIR}/ironic-username")"
+    fi
+  fi
+  if [[ -z "${IRONIC_PASSWORD:-}" ]]; then
+    if [ ! -f "${IRONIC_AUTH_DIR}/ironic-password" ]; then
+      IRONIC_PASSWORD="$(uuid-gen)"
+      echo "${IRONIC_PASSWORD}" > "${IRONIC_AUTH_DIR}/ironic-password"
+    else
+      IRONIC_PASSWORD="$(cat "${IRONIC_AUTH_DIR}/ironic-password")"
+    fi
+  fi
+  IRONIC_INSPECTOR_USERNAME="${IRONIC_INSPECTOR_USERNAME:-${IRONIC_USERNAME}}"
+  IRONIC_INSPECTOR_PASSWORD="${IRONIC_INSPECTOR_PASSWORD:-${IRONIC_PASSWORD}}"
+
+  export IRONIC_USERNAME
+  export IRONIC_PASSWORD
+  export IRONIC_INSPECTOR_USERNAME
+  export IRONIC_INSPECTOR_PASSWORD
+fi
+
+for overlay in "${BMO_OVERLAYS[@]}"; do
+  echo "${IRONIC_USERNAME}" > "${overlay}/ironic-username"
+  echo "${IRONIC_PASSWORD}" > "${overlay}/ironic-password"
+  if [[ "${overlay}" =~ release-0\.[1-5]$ ]]; then
+    echo "${IRONIC_INSPECTOR_USERNAME}" > "${overlay}/ironic-inspector-username"
+    echo "${IRONIC_INSPECTOR_PASSWORD}" > "${overlay}/ironic-inspector-password"
+  fi
+done
+
+for overlay in "${IRONIC_OVERLAYS[@]}"; do
+  echo "IRONIC_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_USERNAME}" "${IRONIC_PASSWORD}")" > \
+    "${overlay}/ironic-htpasswd"
+  envsubst < "${REPO_ROOT}/test/e2e/data/ironic-deployment/components/basic-auth/ironic-auth-config-tpl" > \
+  "${overlay}/ironic-auth-config"
+  IRONIC_INSPECTOR_AUTH_CONFIG_TPL="/tmp/ironic-inspector-auth-config-tpl"
+  curl -o "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" https://raw.githubusercontent.com/metal3-io/baremetal-operator/release-0.5/ironic-deployment/components/basic-auth/ironic-inspector-auth-config-tpl
+  envsubst < "${IRONIC_INSPECTOR_AUTH_CONFIG_TPL}" > \
+    "${overlay}/ironic-inspector-auth-config"
+  echo "INSPECTOR_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_INSPECTOR_USERNAME}" \
+    "${IRONIC_INSPECTOR_PASSWORD}")" > "${overlay}/ironic-inspector-htpasswd"
+done
+
 # run e2e tests
 make e2e-tests

test/e2e/common.go

@@ -23,6 +23,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"golang.org/x/crypto/ssh"
+	v1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -33,8 +34,11 @@ import (
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
 	expv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
+	testexec "sigs.k8s.io/cluster-api/test/framework/exec"
 	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/kustomize/api/filesys"
+	"sigs.k8s.io/kustomize/api/krusty"
 )
 
 type vmState string
@@ -49,6 +53,7 @@ const (
 	ironicImageDir         = "/opt/metal3-dev-env/ironic/html/images"
 	osTypeCentos           = "centos"
 	osTypeUbuntu           = "ubuntu"
+	ironicSuffix           = "-ironic"
 )
 
 func Byf(format string, a ...interface{}) {
@@ -779,3 +784,134 @@ func LabelCRD(ctx context.Context, c client.Client, crdName string, labels map[s
 	Logf("CRD '%s' labeled successfully\n", crdName)
 	return nil
 }
+
+// BuildAndApplyKustomizationInput provides input for BuildAndApplyKustomize().
+// If WaitForDeployment and/or WatchDeploymentLogs is set to true, then DeploymentName
+// and DeploymentNamespace are expected.
+type BuildAndApplyKustomizationInput struct {
+	// Path to the kustomization to build
+	Kustomization string
+
+	ClusterProxy framework.ClusterProxy
+
+	// If this is set to true. Perform a wait until the deployment specified by
+	// DeploymentName and DeploymentNamespace is available or WaitIntervals is timed out
+	WaitForDeployment bool
+
+	// If this is set to true. Set up a log watcher for the deployment specified by
+	// DeploymentName and DeploymentNamespace
+	WatchDeploymentLogs bool
+
+	// DeploymentName and DeploymentNamespace specified a deployment that will be waited and/or logged
+	DeploymentName      string
+	DeploymentNamespace string
+
+	// Path to store the deployment logs
+	LogPath string
+
+	// Intervals to use in checking and waiting for the deployment
+	WaitIntervals []interface{}
+}
+
+func (input *BuildAndApplyKustomizationInput) validate() error {
+	// If neither WaitForDeployment nor WatchDeploymentLogs is true, we don't need to validate the input
+	if !input.WaitForDeployment && !input.WatchDeploymentLogs {
+		return nil
+	}
+	if input.WaitForDeployment && input.WaitIntervals == nil {
+		return fmt.Errorf("WaitIntervals is expected if WaitForDeployment is set to true")
+	}
+	if input.WatchDeploymentLogs && input.LogPath == "" {
+		return fmt.Errorf("LogPath is expected if WatchDeploymentLogs is set to true")
+	}
+	if input.DeploymentName == "" || input.DeploymentNamespace == "" {
+		return fmt.Errorf("DeploymentName and DeploymentNamespace are expected if WaitForDeployment or WatchDeploymentLogs is true")
+	}
+	return nil
+}
+
+// BuildAndApplyKustomization takes input from BuildAndApplyKustomizationInput. It builds the provided kustomization
+// and apply it to the cluster provided by clusterProxy.
+func BuildAndApplyKustomization(ctx context.Context, input *BuildAndApplyKustomizationInput) error {
+	Expect(input.validate()).To(Succeed())
+	var err error
+	kustomization := input.Kustomization
+	clusterProxy := input.ClusterProxy
+	manifest, err := buildKustomizeManifest(kustomization)
+	if err != nil {
+		return err
+	}
+
+	err = clusterProxy.Apply(ctx, manifest)
+	if err != nil {
+		return err
+	}
+
+	if !input.WaitForDeployment && !input.WatchDeploymentLogs {
+		return nil
+	}
+
+	deployment := &v1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      input.DeploymentName,
+			Namespace: input.DeploymentNamespace,
+		},
+	}
+
+	if input.WaitForDeployment {
+		// Wait for the deployment to become available
+		framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{
+			Getter:     clusterProxy.GetClient(),
+			Deployment: deployment,
+		}, input.WaitIntervals...)
+	}
+
+	if input.WatchDeploymentLogs {
+		// Set up log watcher
+		framework.WatchDeploymentLogsByName(ctx, framework.WatchDeploymentLogsByNameInput{
+			GetLister:  clusterProxy.GetClient(),
+			Cache:      clusterProxy.GetCache(ctx),
+			ClientSet:  clusterProxy.GetClientSet(),
+			Deployment: deployment,
+			LogPath:    input.LogPath,
+		})
+	}
+	return nil
+}
+
+// BuildAndRemoveKustomization builds the provided kustomization to resources and removes them from the cluster
+// provided by clusterProxy.
+func BuildAndRemoveKustomization(ctx context.Context, kustomization string, clusterProxy framework.ClusterProxy) error {
+	manifest, err := buildKustomizeManifest(kustomization)
+	if err != nil {
+		return err
+	}
+	return KubectlDelete(ctx, clusterProxy.GetKubeconfigPath(), manifest)
+}
+
+// KubectlDelete shells out to kubectl delete.
+func KubectlDelete(ctx context.Context, kubeconfigPath string, resources []byte, args ...string) error {
+	aargs := append([]string{"delete", "--kubeconfig", kubeconfigPath, "-f", "-"}, args...)
+	rbytes := bytes.NewReader(resources)
+	deleteCmd := testexec.NewCommand(
+		testexec.WithCommand("kubectl"),
+		testexec.WithArgs(aargs...),
+		testexec.WithStdin(rbytes),
+	)
+
+	fmt.Printf("Running kubectl %s\n", strings.Join(aargs, " "))
+	stdout, stderr, err := deleteCmd.Run(ctx)
+	fmt.Printf("stderr:\n%s\n", string(stderr))
+	fmt.Printf("stdout:\n%s\n", string(stdout))
+	return err
+}
+
+func buildKustomizeManifest(source string) ([]byte, error) {
+	kustomizer := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
+	fSys := filesys.MakeFsOnDisk()
+	resources, err := kustomizer.Run(fSys, source)
+	if err != nil {
+		return nil, err
+	}
+	return resources.AsYaml()
+}

test/e2e/config/e2e_conf.yaml

@@ -166,6 +166,9 @@ variables:
   IMAGE_CHECKSUM_TYPE: "sha256"
   IMAGE_USERNAME: "metal3"
   NODE_DRAIN_TIMEOUT: "0s"
+  IRONIC_RELEASE_23.1: "data/ironic-deployment/overlays/release-23.1"
+  IRONIC_RELEASE_24.0: "data/ironic-deployment/overlays/release-24.0"
+  BMO_RELEASE_0.5: "data/bmo-deployment/overlays/release-0.5"
 
 intervals:
   default/wait-controllers: ["5m", "10s"]

test/e2e/data/bmo-deployment/components/basic-auth/credentials_patch.yaml

@@ -0,0 +1,18 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        volumeMounts:
+        - name: ironic-credentials
+          mountPath: "/opt/metal3/auth/ironic"
+          readOnly: true
+      volumes:
+      - name: ironic-credentials
+        secret:
+          secretName: ironic-credentials

test/e2e/data/bmo-deployment/components/basic-auth/kustomization.yaml

@@ -0,0 +1,19 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+# NOTE: This component requires a secret with the basic auth credentials!
+# How you create it is up to you. The required secrets is:
+# - ironic-credentials
+#
+# It should contain 2 fields: username and password. Example:
+#
+# apiVersion: v1
+# kind: Secret
+# metadata:
+#   name: ironic-credentials
+# data:
+#   password: <base64-encoded-password>
+#   username: <base64-encoded-username>
+
+patches:
+- path: credentials_patch.yaml

test/e2e/data/bmo-deployment/components/tls/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+patches:
+- path: tls_ca_patch.yaml
+  target:
+    kind: Deployment
+    name: controller-manager

test/e2e/data/bmo-deployment/components/tls/tls_ca_patch.yaml

@@ -0,0 +1,17 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        volumeMounts:
+          - name: cacert
+            mountPath: "/opt/metal3/certs/ca"
+            readOnly: true
+      volumes:
+      - name: cacert
+        secret:
+          secretName: ironic-cacert

test/e2e/data/bmo-deployment/overlays/release-0.5/ironic.env

@@ -0,0 +1,4 @@
+DEPLOY_KERNEL_URL=http://172.22.0.2:6180/images/ironic-python-agent.kernel
+DEPLOY_RAMDISK_URL=http://172.22.0.2:6180/images/ironic-python-agent.initramfs
+IRONIC_ENDPOINT=https://172.22.0.2:6385/v1/
+IRONIC_INSPECTOR_ENDPOINT=https://172.22.0.2:5050/v1/

test/e2e/data/bmo-deployment/overlays/release-0.5/kustomization.yaml

@@ -0,0 +1,38 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: baremetal-operator-system
+resources:
+- https://github.com/metal3-io/baremetal-operator/config/overlays/basic-auth_tls?ref=release-0.5
+configMapGenerator:
+- name: ironic
+  behavior: create
+  envs:
+  - ironic.env
+patches:
+- patch: |
+    # Don't try to pull again the pre-loaded image
+    - op: replace
+      path: /spec/template/spec/containers/0/imagePullPolicy
+      value: IfNotPresent
+  target:
+    kind: Deployment
+    name: controller-manager
+images:
+- name: quay.io/metal3-io/baremetal-operator
+  newTag: release-0.5
+- name: gcr.io/kubebuilder/kube-rbac-proxy
+  newTag: v0.8.0
+# We cannot use suffix hashes since the kustomizations we build on
+# cannot be aware of what suffixes we add.
+generatorOptions:
+  disableNameSuffixHash: true
+# NOTE: These credentials are generated automatically in hack/ci-e2e.sh
+secretGenerator:
+- name: ironic-credentials
+  files:
+  - username=ironic-username
+  - password=ironic-password
+- name: ironic-inspector-credentials
+  files:
+  - username=ironic-inspector-username
+  - password=ironic-inspector-password

test/e2e/data/ironic-deployment/components/basic-auth/auth.yaml

@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ironic
+spec:
+  template:
+    spec:
+      containers:
+      - name: ironic
+        envFrom:
+        # This is the htpassword matching the ironic password
+        - secretRef:
+            name: ironic-htpasswd
+        - configMapRef:
+            name: ironic-bmo-configmap

test/e2e/data/ironic-deployment/components/basic-auth/ironic-auth-config-tpl

@@ -0,0 +1,4 @@
+[ironic]
+auth_type=http_basic
+username=${IRONIC_USERNAME}
+password=${IRONIC_PASSWORD}