Add crust-gather collection for e2e run

Signed-off-by: Danil Grigorev <danil.grigorev@suse.com>

Makefile

@@ -48,6 +48,8 @@ E2E_DATA_DIR ?= $(ROOT_DIR)/test/e2e/data
 E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/e2e_conf.yaml
 
 export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH)
+export KREW_ROOT := $(abspath $(TOOLS_BIN_DIR))
+export PATH := $(KREW_ROOT)/bin:$(PATH)
 
 # Set --output-base for conversion-gen if we are not within GOPATH
 ifneq ($(abspath $(ROOT_DIR)),$(shell go env GOPATH)/src/github.com/rancher-sandbox/cluster-api-provider-rke2)
@@ -363,6 +365,12 @@ kind-cluster: ## Create a new kind cluster designed for development with Tilt
 tilt-up: kind-cluster ## Start tilt and build kind cluster if needed.
 	tilt up
 
+.PHONY: kubectl
+kubectl: # Download kubectl cli into tools bin folder
+	hack/ensure-kubectl.sh \
+		-b $(TOOLS_BIN_DIR) \
+		$(KUBECTL_VERSION)
+
 ## --------------------------------------
 ## E2E
 ## --------------------------------------
@@ -383,7 +391,7 @@ SKIP_CLEANUP ?= false
 SKIP_CREATE_MGMT_CLUSTER ?= false
 
 .PHONY: test-e2e-run
-test-e2e-run: $(GINKGO) $(KUSTOMIZE) e2e-image inotify-check ## Run the end-to-end tests
+test-e2e-run: $(GINKGO) $(KUSTOMIZE) kubectl e2e-image inotify-check ## Run the end-to-end tests
 	CAPI_KUSTOMIZE_PATH="$(KUSTOMIZE)" time $(GINKGO) -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) \
 	--tags=e2e --focus="$(GINKGO_FOCUS)" -skip="$(GINKGO_SKIP)" --nodes=$(GINKGO_NODES) --no-color=$(GINKGO_NOCOLOR) \
 	--timeout=$(GINKGO_TIMEOUT) --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) ./test/e2e -- \

hack/ensure-kubectl.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+# Copyright © 2023 - 2024 SUSE LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+if [[ "${TRACE-0}" == "1" ]]; then
+    set -o xtrace
+fi
+
+# shellcheck source=./hack/utils.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
+GOPATH_BIN="$(go env GOPATH)/bin/"
+MINIMUM_KUBECTL_VERSION=v1.27.0
+goarch="$(go env GOARCH)"
+goos="$(go env GOOS)"
+
+# Ensure the kubectl tool exists and is a viable version, or installs it
+verify_kubectl_version() {
+
+  # If kubectl is not available on the path, get it
+  if ! [ -x "$(command -v kubectl)" ]; then
+    if [ "$goos" == "linux" ] || [ "$goos" == "darwin" ]; then
+      if ! [ -d "${GOPATH_BIN}" ]; then
+        mkdir -p "${GOPATH_BIN}"
+      fi
+      echo 'kubectl not found, installing'
+      curl -sLo "${GOPATH_BIN}/kubectl" "https://dl.k8s.io/release/${MINIMUM_KUBECTL_VERSION}/bin/${goos}/${goarch}/kubectl"
+      chmod +x "${GOPATH_BIN}/kubectl"
+      verify_gopath_bin
+    else
+      echo "Missing required binary in path: kubectl"
+      return 2
+    fi
+  fi
+
+  local kubectl_version
+  IFS=" " read -ra kubectl_version <<< "$(kubectl version --client)"
+  if [[ "${MINIMUM_KUBECTL_VERSION}" != $(echo -e "${MINIMUM_KUBECTL_VERSION}\n${kubectl_version[2]}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) ]]; then
+    cat <<EOF
+Detected kubectl version: ${kubectl_version[2]}.
+Requires ${MINIMUM_KUBECTL_VERSION} or greater.
+Please install ${MINIMUM_KUBECTL_VERSION} or later.
+EOF
+    return 2
+  fi
+}
+
+install_plugins() {
+  (
+    set -x; cd "$(mktemp -d)" &&
+    OS="$(uname | tr '[:upper:]' '[:lower:]')" &&
+    ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" &&
+    KREW="krew-${OS}_${ARCH}" &&
+    curl -fsSLO "https://github.com/kubernetes-sigs/krew/releases/latest/download/${KREW}.tar.gz" &&
+    tar zxvf "${KREW}.tar.gz" &&
+    ./"${KREW}" install krew
+  )
+  kubectl krew index add crust-gather https://github.com/crust-gather/crust-gather.git || true
+  kubectl krew install crust-gather/crust-gather
+}
+
+verify_kubectl_version
+install_plugins

hack/utils.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Copyright © 2023 - 2024 SUSE LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# get_root_path returns the root path of the project source tree
+get_root_path() {
+    git rev-parse --show-toplevel
+}
+
+# cd_root_path cds to the root path of the project source tree
+cd_root_path() {
+    cd "$(get_root_path)" || exit
+}
+
+# ensure GOPATH/bin is in PATH as we may install binaries to that directory in
+# other ensure-* scripts, and expect them to be found in PATH later on
+verify_gopath_bin() {
+    local gopath_bin
+
+    gopath_bin="$(go env GOPATH)/bin"
+    if ! printenv PATH | grep -q "${gopath_bin}"; then
+        cat <<EOF
+error: \$GOPATH/bin=${gopath_bin} is not in your PATH.
+See https://go.dev/doc/gopath_code for more instructions.
+EOF
+        return 2
+    fi
+}
+

test/e2e/e2e_test.go

@@ -72,6 +72,9 @@ var _ = Describe("Workload cluster creation", func() {
 	})
 
 	AfterEach(func() {
+		err := CollectArtifacts(ctx, bootstrapClusterProxy.GetKubeconfigPath(), filepath.Join(artifactFolder, bootstrapClusterProxy.GetName(), clusterName+specName))
+		Expect(err).ToNot(HaveOccurred())
+
 		cleanInput := cleanupInput{
 			SpecName:          specName,
 			Cluster:           result.Cluster,

test/e2e/e2e_upgrade_test.go

@@ -69,6 +69,9 @@ var _ = Describe("Workload cluster creation", func() {
 	})
 
 	AfterEach(func() {
+		err := CollectArtifacts(ctx, bootstrapClusterProxy.GetKubeconfigPath(), filepath.Join(artifactFolder, bootstrapClusterProxy.GetName(), clusterName+specName))
+		Expect(err).ToNot(HaveOccurred())
+
 		cleanInput := cleanupInput{
 			SpecName:          specName,
 			Cluster:           result.Cluster,

test/e2e/helpers.go

@@ -19,6 +19,7 @@ limitations under the License.
 package e2e
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -433,6 +434,32 @@ func setDefaults(input *ApplyClusterTemplateAndWaitInput) {
 	}
 }
 
+var secrets = []string{}
+
+func CollectArtifacts(ctx context.Context, kubeconfigPath, name string, args ...string) error {
+	if kubeconfigPath == "" {
+		return fmt.Errorf("Unable to collect artifacts: kubeconfig path is empty")
+	}
+
+	aargs := append([]string{"crust-gather", "collect", "--kubeconfig", kubeconfigPath, "-v", "ERROR", "-f", name}, args...)
+	for _, secret := range secrets {
+		aargs = append(aargs, "-s", secret)
+	}
+
+	cmd := exec.Command("kubectl", aargs...)
+
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	cmd.WaitDelay = time.Minute
+
+	fmt.Printf("Running kubectl %s\n", strings.Join(aargs, " "))
+	err := cmd.Run()
+	fmt.Printf("stderr:\n%s\n", string(stderr.Bytes()))
+	fmt.Printf("stdout:\n%s\n", string(stdout.Bytes()))
+	return err
+}
+
 type networks []network
 
 type network struct {

test/e2e/metadata_e2e_test.go

@@ -76,6 +76,9 @@ var _ = Describe("Workload cluster creation", func() {
 	})
 
 	AfterEach(func() {
+		err := CollectArtifacts(ctx, bootstrapClusterProxy.GetKubeconfigPath(), filepath.Join(artifactFolder, bootstrapClusterProxy.GetName(), clusterName+specName))
+		Expect(err).ToNot(HaveOccurred())
+
 		cleanInput := cleanupInput{
 			SpecName:          specName,
 			Cluster:           result.Cluster,