Adds E2E coverage reporting

.github/workflows/e2e.yaml

@@ -22,4 +22,17 @@ jobs:
 
     - name: Run e2e tests
       run: |
-        make e2e
+        # By default make stops building on first non-zero exit code which
+        # in case of E2E tests will mean that code coverage will only be
+        # collected on successful runs. We want to collect coverage even
+        # after failing tests.
+        # With -k flag make will continue the build, but will return non-zero
+        # exit code in case of any errors.
+        make -k e2e
+
+    - uses: codecov/codecov-action@v3
+      with:
+        files: e2e-cover.out
+        flags: e2e
+        fail_ci_if_error: true
+        functionalities: fixes

.github/workflows/unit-test.yaml

@@ -25,5 +25,6 @@ jobs:
     - uses: codecov/codecov-action@v3
       with:
         files: cover.out
+        flags: unit
         fail_ci_if_error: true
         functionalities: fixes

.gitignore

@@ -14,8 +14,9 @@ Dockerfile.cross
 # Test binary, build with `go test -c`
 *.test
 
-# Output of the go coverage tool, specifically when used with LiteIDE
+# Output of the go coverage tools
 *.out
+coverage
 
 # Release output
 dist/**
@@ -35,4 +36,4 @@ install.sh
 .\#*
 
 # documentation website asset folder
-docs/_site
+docs/_site

Makefile

@@ -24,6 +24,8 @@ KIND_CLUSTER_NAME ?= operator-controller
 
 CONTAINER_RUNTIME ?= docker
 
+KUSTOMIZE_BUILD_DIR ?= config/default
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -103,7 +105,13 @@ test-unit: $(SETUP_ENVTEST) ## Run the unit tests
 	eval $$($(SETUP_ENVTEST) use -p env $(ENVTEST_VERSION)) && go test -tags $(GO_BUILD_TAGS) -count=1 -short $(UNIT_TEST_DIRS) -coverprofile cover.out
 
 e2e: KIND_CLUSTER_NAME=operator-controller-e2e
-e2e: run kind-load-test-artifacts test-e2e kind-cluster-cleanup ## Run e2e test suite on local kind cluster
+e2e: KUSTOMIZE_BUILD_DIR=config/e2e
+e2e: GO_BUILD_FLAGS=-cover
+e2e: run kind-load-test-artifacts test-e2e e2e-coverage kind-cluster-cleanup ## Run e2e test suite on local kind cluster
+
+.PHONY: e2e-coverage
+e2e-coverage:
+	COVERAGE_OUTPUT=./e2e-cover.out ./hack/e2e-coverage.sh
 
 kind-load: $(KIND) ## Loads the currently constructed image onto the cluster
 	$(KIND) load docker-image $(IMG) --name $(KIND_CLUSTER_NAME)
@@ -115,6 +123,7 @@ kind-cluster: $(KIND) kind-cluster-cleanup ## Standup a kind cluster
 kind-cluster-cleanup: $(KIND) ## Delete the kind cluster
 	$(KIND) delete cluster --name ${KIND_CLUSTER_NAME}
 
+.PHONY: kind-load-test-artifacts
 kind-load-test-artifacts: $(KIND) ## Load the e2e testdata container images into a kind cluster
 	$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/registry-v1/prometheus-operator.v0.47.0 -t localhost/testdata/bundles/registry-v1/prometheus-operator:v0.47.0
 	$(CONTAINER_RUNTIME) build $(TESTDATA_DIR)/bundles/plain-v0/plain.v0.1.0 -t localhost/testdata/bundles/plain-v0/plain:v0.1.0
@@ -131,8 +140,9 @@ export GO_BUILD_ASMFLAGS ?= all=-trimpath=${PWD}
 export GO_BUILD_LDFLAGS  ?= -s -w -X $(shell go list -m)/version.Version=$(VERSION)
 export GO_BUILD_GCFLAGS  ?= all=-trimpath=${PWD}
 export GO_BUILD_TAGS     ?= upstream
+export GO_BUILD_FLAGS    ?=
 
-BUILDCMD = go build -tags '$(GO_BUILD_TAGS)' -ldflags '$(GO_BUILD_LDFLAGS)' -gcflags '$(GO_BUILD_GCFLAGS)' -asmflags '$(GO_BUILD_ASMFLAGS)' -o $(BUILDBIN)/manager ./cmd/manager
+BUILDCMD = go build $(GO_BUILD_FLAGS) -tags '$(GO_BUILD_TAGS)' -ldflags '$(GO_BUILD_LDFLAGS)' -gcflags '$(GO_BUILD_GCFLAGS)' -asmflags '$(GO_BUILD_ASMFLAGS)' -o $(BUILDBIN)/manager ./cmd/manager
 
 .PHONY: build-deps
 build-deps: manifests generate fmt vet
@@ -173,7 +183,7 @@ release: $(GORELEASER) ## Runs goreleaser for the operator-controller. By defaul
 
 quickstart: export MANIFEST="https://github.com/operator-framework/operator-controller/releases/download/$(VERSION)/operator-controller.yaml"
 quickstart: $(KUSTOMIZE) generate ## Generate the installation release manifests and scripts
-	$(KUSTOMIZE) build config/default | sed "s/:devel/:$(VERSION)/g" > operator-controller.yaml
+	$(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) | sed "s/:devel/:$(VERSION)/g" > operator-controller.yaml
 	envsubst '$$CATALOGD_VERSION,$$CERT_MGR_VERSION,$$RUKPAK_VERSION,$$MANIFEST' < scripts/install.tpl.sh > install.sh
 
 ##@ Deployment
@@ -185,7 +195,7 @@ endif
 .PHONY: install
 install: export MANIFEST="./operator-controller.yaml"
 install: manifests $(KUSTOMIZE) generate ## Install CRDs into the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/default > operator-controller.yaml
+	$(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) > operator-controller.yaml
 	envsubst '$$CATALOGD_VERSION,$$CERT_MGR_VERSION,$$RUKPAK_VERSION,$$MANIFEST' < scripts/install.tpl.sh | bash -s
 
 .PHONY: uninstall
@@ -195,8 +205,8 @@ uninstall: manifests $(KUSTOMIZE) ## Uninstall CRDs from the K8s cluster specifi
 .PHONY: deploy
 deploy: manifests $(KUSTOMIZE) ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
-	$(KUSTOMIZE) build config/default | kubectl apply -f -
+	$(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) | kubectl apply -f -
 
 .PHONY: undeploy
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
-	$(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
+	$(KUSTOMIZE) build $(KUSTOMIZE_BUILD_DIR) | kubectl delete --ignore-not-found=$(ignore-not-found) -f -

config/e2e/kustomization.yaml

@@ -0,0 +1,9 @@
+namespace: operator-controller-system
+
+resources:
+- ../default
+- manager_e2e_coverage_pvc.yaml
+- manager_e2e_coverage_copy_pod.yaml
+
+patches:
+- path: manager_e2e_coverage_patch.yaml

config/e2e/manager_e2e_coverage_copy_pod.yaml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: e2e-coverage-copy-pod
+  labels:
+    app.kubernetes.io/name: e2e-coverage-copy-pod
+    app.kubernetes.io/instance: controller-manager
+    app.kubernetes.io/component: e2e-coverage
+    app.kubernetes.io/created-by: operator-controller
+    app.kubernetes.io/part-of: operator-controller
+    app.kubernetes.io/managed-by: kustomize
+spec:
+  restartPolicy: Never
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 65532
+    seccompProfile:
+      type: RuntimeDefault
+  containers:
+  - name: tar
+    image: busybox:1.36
+    command: ["sleep", "infinity"]
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+          - "ALL"
+    volumeMounts:
+    - name: e2e-coverage-volume
+      mountPath: /e2e-coverage
+      readOnly: true
+  volumes:
+  - name: e2e-coverage-volume
+    persistentVolumeClaim:
+      claimName: e2e-coverage
+      readOnly: true

config/e2e/manager_e2e_coverage_patch.yaml

@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: kube-rbac-proxy
+      - name: manager
+        env:
+        - name: GOCOVERDIR
+          value: /e2e-coverage
+        volumeMounts:
+        - name: e2e-coverage-volume
+          mountPath: /e2e-coverage
+      volumes:
+      - name: e2e-coverage-volume
+        persistentVolumeClaim:
+          claimName: e2e-coverage

config/e2e/manager_e2e_coverage_pvc.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: e2e-coverage
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 64Mi

hack/e2e-coverage.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -euo pipefail
+
+COVERAGE_OUTPUT="${COVERAGE_OUTPUT:-e2e-cover.out}"
+
+OPERATOR_CONTROLLER_NAMESPACE="operator-controller-system"
+OPERATOR_CONTROLLER_MANAGER_DEPLOYMENT_NAME="operator-controller-controller-manager"
+COPY_POD_NAME="e2e-coverage-copy-pod"
+
+# Create a temporary directory for coverage
+COVERAGE_DIR=$(mktemp -d)
+
+# Trap to cleanup temporary directory on script exit
+function cleanup {
+    rm -rf "$COVERAGE_DIR"
+}
+trap cleanup EXIT
+
+# Coverage-instrumented binary produces coverage on termination,
+# so we scale down the manager before gathering the coverage
+kubectl -n "$OPERATOR_CONTROLLER_NAMESPACE" scale deployment/"$OPERATOR_CONTROLLER_MANAGER_DEPLOYMENT_NAME" --replicas=0
+
+# Wait for the copy pod to be ready
+kubectl -n "$OPERATOR_CONTROLLER_NAMESPACE" wait --for=condition=ready pod "$COPY_POD_NAME"
+
+# Copy the coverage data from the temporary pod
+kubectl -n "$OPERATOR_CONTROLLER_NAMESPACE" cp "$COPY_POD_NAME":/e2e-coverage/ "$COVERAGE_DIR"
+
+# Convert binary coverage data files into the textual format
+go tool covdata textfmt -i "$COVERAGE_DIR" -o "$COVERAGE_OUTPUT"
+
+echo "Coverage report generated successfully at: $COVERAGE_OUTPUT"