Makefile: fix issues with repeated evaluations

.github/workflows/e2e.yaml

@@ -16,4 +16,18 @@ jobs:
       - uses: actions/setup-go@v5
         with:
           go-version-file: "go.mod"
-      - run: make e2e
+      - name: Run e2e tests
+        run: |
+          # By default make stops building on first non-zero exit code which
+          # in case of E2E tests will mean that code coverage will only be
+          # collected on successful runs. We want to collect coverage even
+          # after failing tests.
+          # With -k flag make will continue the build, but will return non-zero
+          # exit code in case of any errors.
+          ARTIFACT_PATH=/tmp/artifacts make test-e2e
+
+      - uses: cytopia/upload-artifact-retry-action@v0.1.7
+        if: failure()
+        with:
+          name: e2e-artifacts
+          path: /tmp/artifacts/

.goreleaser.yml

@@ -9,8 +9,6 @@ builds:
     asmflags: "{{ .Env.GO_BUILD_ASMFLAGS }}"
     gcflags: "{{ .Env.GO_BUILD_GCFLAGS }}"
     ldflags: "{{ .Env.GO_BUILD_LDFLAGS }}"
-    tags:
-    - "{{ .Env.GO_BUILD_TAGS }}"
     mod_timestamp: "{{ .CommitTimestamp }}"
     goos:
     - linux

Makefile

@@ -1,6 +1,14 @@
-export IMAGE_REPO                ?= quay.io/operator-framework/catalogd
-export IMAGE_TAG                 ?= devel
-IMAGE=$(IMAGE_REPO):$(IMAGE_TAG)
+ifeq ($(origin IMAGE_REPO), undefined)
+IMAGE_REPO := quay.io/operator-framework/catalogd
+endif
+export IMAGE_REPO
+
+ifeq ($(origin IMAGE_TAG), undefined)
+IMAGE_TAG := devel
+endif
+export IMAGE_TAG
+
+IMAGE := $(IMAGE_REPO):$(IMAGE_TAG)
 
 # setup-envtest on *nix uses XDG_DATA_HOME, falling back to HOME, as the default storage directory. Some CI setups
 # don't have XDG_DATA_HOME set; in those cases, we set it here so setup-envtest functions correctly. This shouldn't
@@ -11,17 +19,13 @@ export XDG_DATA_HOME ?= /tmp/.local/share
 include .bingo/Variables.mk
 
 # Dependencies
-CERT_MGR_VERSION        ?= v1.11.0
-ENVTEST_SERVER_VERSION = $(shell go list -m k8s.io/client-go | cut -d" " -f2 | sed 's/^v0\.\([[:digit:]]\{1,\}\)\.[[:digit:]]\{1,\}$$/1.\1.x/')
+export CERT_MGR_VERSION := v1.11.0
+ENVTEST_SERVER_VERSION := $(shell go list -m k8s.io/client-go | cut -d" " -f2 | sed 's/^v0\.\([[:digit:]]\{1,\}\)\.[[:digit:]]\{1,\}$$/1.\1.x/')
 
 # Cluster configuration
-KIND_CLUSTER_NAME       ?= catalogd
-CATALOGD_NAMESPACE      ?= catalogd-system
-KIND_CLUSTER_IMAGE      ?= kindest/node:v1.28.0@sha256:b7a4cad12c197af3ba43202d3efe03246b3f0793f162afb40a33c923952d5b31
-
-# E2E configuration
-TESTDATA_DIR            ?= testdata
-
+KIND_CLUSTER_NAME := catalogd
+CATALOGD_NAMESPACE := catalogd-system
+KIND_CLUSTER_IMAGE := kindest/node:v1.28.0@sha256:b7a4cad12c197af3ba43202d3efe03246b3f0793f162afb40a33c923952d5b31
 
 ##@ General
 
@@ -56,16 +60,18 @@ fmt: ## Run go fmt against code.
 
 .PHONY: vet
 vet: ## Run go vet against code.
-	go vet -tags '$(GO_BUILD_TAGS)' ./...
+	go vet ./...
 
 .PHONY: test-unit
 test-unit: generate fmt vet $(SETUP_ENVTEST) ## Run tests.
 	eval $$($(SETUP_ENVTEST) use -p env $(ENVTEST_SERVER_VERSION)) && go test $(shell go list ./... | grep -v /test/e2e) -coverprofile cover.out
 
 FOCUS := $(if $(TEST),-v -focus "$(TEST)")
-E2E_FLAGS ?= ""
+ifeq ($(origin E2E_FLAGS), undefined)
+E2E_FLAGS := ""
+endif
 test-e2e: $(GINKGO) ## Run the e2e tests
-	$(GINKGO) --tags $(GO_BUILD_TAGS) $(E2E_FLAGS) -trace -vv $(FOCUS) test/e2e
+	$(GINKGO) $(E2E_FLAGS) -trace -vv $(FOCUS) test/e2e
 
 e2e: KIND_CLUSTER_NAME=catalogd-e2e
 e2e: run image-registry test-e2e kind-cluster-cleanup ## Run e2e test suite on local kind cluster
@@ -91,39 +97,37 @@ BINARIES=manager
 LINUX_BINARIES=$(join $(addprefix linux/,$(BINARIES)), )
 
 # Build info
-export VERSION_PKG     ?= $(shell go list -m)/internal/version
+export VERSION_PKG     := $(shell go list -m)/internal/version
 
-export GIT_COMMIT      ?= $(shell git rev-parse HEAD)
-export GIT_VERSION     ?= $(shell git describe --tags --always --dirty)
-export GIT_TREE_STATE  ?= $(shell [ -z "$(shell git status --porcelain)" ] && echo "clean" || echo "dirty")
-export GIT_COMMIT_DATE ?= $(shell TZ=UTC0 git show --quiet --date=format:'%Y-%m-%dT%H:%M:%SZ' --format="%cd")
+export GIT_COMMIT      := $(shell git rev-parse HEAD)
+export GIT_VERSION     := $(shell git describe --tags --always --dirty)
+export GIT_TREE_STATE  := $(shell [ -z "$(shell git status --porcelain)" ] && echo "clean" || echo "dirty")
+export GIT_COMMIT_DATE := $(shell TZ=UTC0 git show --quiet --date=format:'%Y-%m-%dT%H:%M:%SZ' --format="%cd")
 
-export CGO_ENABLED       ?= 0
-export GO_BUILD_ASMFLAGS ?= all=-trimpath=${PWD}
-export GO_BUILD_LDFLAGS  ?= -s -w \
+export GO_BUILD_ASMFLAGS := all=-trimpath=${PWD}
+export GO_BUILD_LDFLAGS  := -s -w \
     -X "$(VERSION_PKG).gitVersion=$(GIT_VERSION)" \
     -X "$(VERSION_PKG).gitCommit=$(GIT_COMMIT)" \
     -X "$(VERSION_PKG).gitTreeState=$(GIT_TREE_STATE)" \
     -X "$(VERSION_PKG).commitDate=$(GIT_COMMIT_DATE)"
-export GO_BUILD_GCFLAGS  ?= all=-trimpath=${PWD}
-export GO_BUILD_TAGS     ?=
+export GO_BUILD_GCFLAGS  := all=-trimpath=${PWD}
 
-BUILDCMD = go build -tags '$(GO_BUILD_TAGS)' -ldflags '$(GO_BUILD_LDFLAGS)' -gcflags '$(GO_BUILD_GCFLAGS)' -asmflags '$(GO_BUILD_ASMFLAGS)' -o $(BUILDBIN)/$(notdir $@) ./cmd/$(notdir $@)
+BUILDCMD = go build -ldflags '$(GO_BUILD_LDFLAGS)' -gcflags '$(GO_BUILD_GCFLAGS)' -asmflags '$(GO_BUILD_ASMFLAGS)' -o $(BUILDBIN)/$(notdir $@) ./cmd/$(notdir $@)
 
 .PHONY: build-deps
 build-deps: generate fmt vet
 
 .PHONY: build go-build-local $(BINARIES)
 build: build-deps go-build-local ## Build binaries for current GOOS and GOARCH.
 go-build-local: $(BINARIES)
-$(BINARIES): BUILDBIN = bin
+$(BINARIES): BUILDBIN := bin
 $(BINARIES):
 	$(BUILDCMD)
 
 .PHONY: build-linux go-build-linux $(LINUX_BINARIES)
 build-linux: build-deps go-build-linux ## Build binaries for GOOS=linux and local GOARCH.
 go-build-linux: $(LINUX_BINARIES)
-$(LINUX_BINARIES): BUILDBIN = bin/linux
+$(LINUX_BINARIES): BUILDBIN := bin/linux
 $(LINUX_BINARIES):
 	GOOS=linux $(BUILDCMD)
 
@@ -164,13 +168,19 @@ undeploy: $(KUSTOMIZE) ## Undeploy Catalogd from the K8s cluster specified in ~/
 	$(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=true -f -
 
 wait:
-	kubectl wait --for=condition=Available --namespace=$(CATALOGD_NAMESPACE) deployment/catalogd-controller-manager --timeout=60s
+	kubectl wait --for=condition=Available --namespace=$(CATALOGD_NAMESPACE) deployment/catalogd-controller-manager --timeout=60s || (kubectl describe pods --namespace $(CATALOGD_NAMESPACE) && kubectl logs --namespace=$(CATALOGD_NAMESPACE) deployments/catalogd-controller-manager)
 
 ##@ Release
 
-export ENABLE_RELEASE_PIPELINE ?= false
-export GORELEASER_ARGS         ?= --snapshot --clean
-export CERT_MGR_VERSION        ?= $(CERT_MGR_VERSION)
+ifeq ($(origin ENABLE_RELEASE_PIPELINE), undefined)
+ENABLE_RELEASE_PIPELINE := false
+endif
+export ENABLE_RELEASE_PIPELINE
+
+ifeq ($(origin GORELEASER_ARGS), undefined)
+GORELEASER_ARGS := --snapshot --clean
+endif
+
 release: $(GORELEASER) ## Runs goreleaser for catalogd. By default, this will run only as a snapshot and will not publish any artifacts unless it is run with different arguments. To override the arguments, run with "GORELEASER_ARGS=...". When run as a github action from a tag, this target will publish a full release.
 	$(GORELEASER) $(GORELEASER_ARGS)
 
@@ -180,4 +190,3 @@ quickstart: $(KUSTOMIZE) generate ## Generate the installation release manifests
 .PHONY: demo-update
 demo-update:
 	hack/scripts/generate-asciidemo.sh
-

test/e2e/e2e_suite_test.go

@@ -1,17 +1,28 @@
 package e2e
 
 import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
 	"testing"
 	"time"
 
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
-
+	"github.com/stretchr/testify/require"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
+	"k8s.io/utils/env"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/yaml"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
 
 	catalogd "github.com/operator-framework/catalogd/api/core/v1alpha1"
 )
@@ -27,6 +38,9 @@ func TestE2E(t *testing.T) {
 	RegisterFailHandler(Fail)
 	SetDefaultEventuallyTimeout(1 * time.Minute)
 	SetDefaultEventuallyPollingInterval(1 * time.Second)
+	defer func() {
+		require.NoError(t, getArtifactsOutput())
+	}()
 	RunSpecs(t, "E2E Suite")
 }
 
@@ -40,3 +54,113 @@ var _ = BeforeSuite(func() {
 	kubeClient, err = kubernetes.NewForConfig(cfg)
 	Expect(err).ToNot(HaveOccurred())
 })
+
+// getArtifactsOutput gets all the artifacts from the test run and saves them to the artifact path.
+// Currently it saves:
+// - pods logs
+// - deployments
+// - catalogs
+func getArtifactsOutput() error {
+	basePath := env.GetString("ARTIFACT_PATH", "")
+	if basePath == "" {
+		return nil
+	}
+	if kubeClient == nil {
+		return fmt.Errorf("kubeClient is nil")
+	}
+
+	// Get the test description and sanitize it for use as a directory name
+	artifactPath := filepath.Join(basePath, "catalogd-e2e", fmt.Sprint(time.Now().UnixNano()))
+
+	// Create the full artifact path
+	if err := os.MkdirAll(artifactPath, 0755); err != nil {
+		return err
+	}
+
+	// Get all namespaces
+	namespaces := corev1.NamespaceList{}
+	if err := c.List(context.Background(), &namespaces); err != nil {
+		fmt.Printf("Failed to list namespaces: %v", err)
+	}
+
+	// get all catalogs save them to the artifact path.
+	catalogs := catalogd.CatalogList{}
+	if err := c.List(context.Background(), &catalogs, client.InNamespace("")); err != nil {
+		fmt.Printf("Failed to list catalogs: %v", err)
+	}
+	for _, catalog := range catalogs.Items {
+		// Save catalog to artifact path
+		catalogYaml, err := yaml.Marshal(catalog)
+		if err != nil {
+			fmt.Printf("Failed to marshal catalog: %v", err)
+			continue
+		}
+		if err := os.WriteFile(filepath.Join(artifactPath, catalog.Name+"-catalog.yaml"), catalogYaml, 0600); err != nil {
+			fmt.Printf("Failed to write catalog to file: %v", err)
+		}
+	}
+
+	for _, namespace := range namespaces.Items {
+		// let's ignore kube-* namespaces.
+		if strings.Contains(namespace.Name, "kube-") {
+			continue
+		}
+
+		namespacedArtifactPath := filepath.Join(artifactPath, namespace.Name)
+		if err := os.Mkdir(namespacedArtifactPath, 0755); err != nil {
+			fmt.Printf("Failed to create namespaced artifact path: %v", err)
+			continue
+		}
+
+		// get all deployments in the namespace and save them to the artifact path.
+		deployments := appsv1.DeploymentList{}
+		if err := c.List(context.Background(), &deployments, client.InNamespace(namespace.Name)); err != nil {
+			fmt.Printf("Failed to list deployments %v in namespace: %q", err, namespace.Name)
+			continue
+		}
+
+		for _, deployment := range deployments.Items {
+			// Save deployment to artifact path
+			deploymentYaml, err := yaml.Marshal(deployment)
+			if err != nil {
+				fmt.Printf("Failed to marshal deployment: %v", err)
+				continue
+			}
+			if err := os.WriteFile(filepath.Join(namespacedArtifactPath, deployment.Name+"-deployment.yaml"), deploymentYaml, 0600); err != nil {
+				fmt.Printf("Failed to write deployment to file: %v", err)
+			}
+		}
+
+		// Get logs from all pods in all namespaces
+		pods := corev1.PodList{}
+		if err := c.List(context.Background(), &pods, client.InNamespace(namespace.Name)); err != nil {
+			fmt.Printf("Failed to list pods %v in namespace: %q", err, namespace.Name)
+		}
+		for _, pod := range pods.Items {
+			if pod.Status.Phase != corev1.PodRunning && pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed {
+				continue
+			}
+			for _, container := range pod.Spec.Containers {
+				logs, err := kubeClient.CoreV1().Pods(namespace.Name).GetLogs(pod.Name, &corev1.PodLogOptions{Container: container.Name}).Stream(context.Background())
+				if err != nil {
+					fmt.Printf("Failed to get logs for pod %q in namespace %q: %v", pod.Name, namespace.Name, err)
+					continue
+				}
+				defer logs.Close()
+
+				outFile, err := os.Create(filepath.Join(namespacedArtifactPath, pod.Name+"-"+container.Name+"-logs.txt"))
+				if err != nil {
+					fmt.Printf("Failed to create file for pod %q in namespace %q: %v", pod.Name, namespace.Name, err)
+					continue
+				}
+				defer outFile.Close()
+
+				if _, err := io.Copy(outFile, logs); err != nil {
+					fmt.Printf("Failed to copy logs for pod %q in namespace %q: %v", pod.Name, namespace.Name, err)
+					continue
+				}
+			}
+		}
+	}
+	return nil
+}

test/e2e/unpack_test.go

@@ -2,19 +2,17 @@ package e2e
 
 import (
 	"context"
-	"io"
-	"net/url"
-	"os"
-	"strings"
-
+	"github.com/google/go-cmp/cmp"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-
-	"github.com/google/go-cmp/cmp"
+	"io"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"net/url"
+	"os"
+	"strings"
 
 	catalogd "github.com/operator-framework/catalogd/api/core/v1alpha1"
 )