Update templates, use daemonset for kube-vip, start e2e test for updg…

…rade involving kube-vip
kubernetes-sigs · Sep 2, 2021 · e6dd6cb · e6dd6cb
1 parent 8e220a6
commit e6dd6cb
Show file tree

Hide file tree

Showing 44 changed files with 3,089 additions and 4,840 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -98,7 +98,6 @@ jobs:
         E2E_CONF_FILE_SOURCE: "${{ github.workspace }}/test/e2e/config/packet-ci-actions.yaml"
         SKIP_IMAGE_BUILD: "1"
         MANIFEST_PATH: "../../../out/release"
-        ADDONS_PATH: "../../templates/addons"
       run: ./scripts/ci-e2e-capi${{ matrix.category }}.sh
     - name: Upload artifact
       uses: actions/upload-artifact@v2

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -38,8 +38,8 @@ jobs:
     - name: build e2e artifacts
       env:
         TAG: ${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.version'] }}
-        MANIFEST_PATH: ".."
-        ADDONS_PATH: "./addons"
+        CNI_PATH: "./addons/calico.yaml"
+        CPEM_PATH: "./addons/cpem-v3.22.yaml"
       run: make e2e-artifacts
     - name: build image
       uses: docker/build-push-action@v2

diff --git a/Makefile b/Makefile
@@ -164,13 +164,12 @@ e2e-artifacts: clean-e2e-artifacts $(E2E_ARTIFACTS_DIR) $(ENVSUBST) $(KUSTOMIZE)
 	$(MAKE) set-manifest-image MANIFEST_IMG=$(REGISTRY)/$(IMAGE_NAME) MANIFEST_TAG=$(TAG)
 	$(MAKE) set-manifest-pull-policy PULL_POLICY=IfNotPresent
 	$(MAKE) release-manifests release-metadata e2e-test-templates $(E2E_ARTIFACTS_CONF_FILE) RELEASE_DIR=$(E2E_ARTIFACTS_DIR) TEST_TEMPLATES_TARGET_DIR=$(E2E_ARTIFACTS_DIR)/data E2E_CONF_FILE_SOURCE=$(E2E_ARTIFACTS_CONF_FILE_SOURCE) E2E_CONF_FILE=$(E2E_ARTIFACTS_CONF_FILE)
-	mkdir -p $(E2E_ARTIFACTS_DIR)/addons
-	cp templates/addons/calico.yaml $(E2E_ARTIFACTS_DIR)/addons/
+	cp -r templates/addons $(E2E_ARTIFACTS_DIR)/
 	mkdir -p $(E2E_ARTIFACTS_DIR)/data/kubetest
 	cp test/e2e/data/kubetest/conformance.yaml $(E2E_ARTIFACTS_DIR)/data/kubetest/
 	mkdir -p $(E2E_ARTIFACTS_DIR)/data/shared/v1alpha4
 	cp test/e2e/data/shared/v1alpha4/metadata.yaml $(E2E_ARTIFACTS_DIR)/data/shared/v1alpha4/
-	$(GINKGO) build -tags=e2e ./test/e2e
+	cd test/e2e; $(GINKGO) build -tags=e2e ./
 	mv test/e2e/e2e.test $(E2E_ARTIFACTS_DIR)/
 
 .PHONY: clean-e2e-artifacts
@@ -210,24 +209,23 @@ test-e2e-ci:
 ## E2E Test Templates
 ## --------------------------------------
 
-TEST_TEMPLATES_SOURCE_DIR ?= $(REPO_ROOT)/test/e2e/data
-TEST_TEMPLATES_TARGET_DIR ?= $(TEST_TEMPLATES_SOURCE_DIR)
+TEST_TEMPLATES_TARGET_DIR ?= $(REPO_ROOT)/test/e2e/data
 
 .PHONY: e2e-test-templates
 e2e-test-templates: $(KUSTOMIZE) e2e-test-templates-v1alpha3 e2e-test-templates-v1alpha4 ## Generate cluster templates for all versions
 
 e2e-test-templates-v1alpha3: $(KUSTOMIZE) ## Generate cluster templates for v1alpha3
 	mkdir -p $(TEST_TEMPLATES_TARGET_DIR)/v1alpha3/
-	$(KUSTOMIZE) build $(TEST_TEMPLATES_SOURCE_DIR)/v1alpha3/cluster-template --load_restrictor none > $(TEST_TEMPLATES_TARGET_DIR)/v1alpha3/cluster-template.yaml
+	$(KUSTOMIZE) build $(REPO_ROOT)/test/e2e/data/v1alpha3/cluster-template --load_restrictor none > $(TEST_TEMPLATES_TARGET_DIR)/v1alpha3/cluster-template.yaml
 
 e2e-test-templates-v1alpha4: $(KUSTOMIZE) ## Generate cluster templates for v1alpha4
 	mkdir -p $(TEST_TEMPLATES_TARGET_DIR)/v1alpha4/
-	$(KUSTOMIZE) build $(TEST_TEMPLATES_SOURCE_DIR)/v1alpha4/cluster-template --load_restrictor none > $(TEST_TEMPLATES_TARGET_DIR)/v1alpha4/cluster-template.yaml
-	$(KUSTOMIZE) build $(TEST_TEMPLATES_SOURCE_DIR)/v1alpha4/cluster-template-no-cpem --load_restrictor none > $(TEST_TEMPLATES_TARGET_DIR)/v1alpha4/cluster-template-no-cpem.yaml
-	$(KUSTOMIZE) build $(TEST_TEMPLATES_SOURCE_DIR)/v1alpha4/cluster-template-kcp-scale-in --load_restrictor none > $(TEST_TEMPLATES_TARGET_DIR)/v1alpha4/cluster-template-kcp-scale-in.yaml
-	$(KUSTOMIZE) build $(TEST_TEMPLATES_SOURCE_DIR)/v1alpha4/cluster-template-node-drain --load_restrictor none > $(TEST_TEMPLATES_TARGET_DIR)/v1alpha4/cluster-template-node-drain.yaml
-	$(KUSTOMIZE) build $(TEST_TEMPLATES_SOURCE_DIR)/v1alpha4/cluster-template-md-remediation --load_restrictor none > $(TEST_TEMPLATES_TARGET_DIR)/v1alpha4/cluster-template-md-remediation.yaml
-	$(KUSTOMIZE) build $(TEST_TEMPLATES_SOURCE_DIR)/v1alpha4/cluster-template-kcp-remediation --load_restrictor none > $(TEST_TEMPLATES_TARGET_DIR)/v1alpha4/cluster-template-kcp-remediation.yaml
+	$(KUSTOMIZE) build $(REPO_ROOT)/templates/crs-cni-cpem --load_restrictor none > $(TEST_TEMPLATES_TARGET_DIR)/v1alpha4/cluster-template.yaml
+	$(KUSTOMIZE) build $(REPO_ROOT)/templates/no-cloud-provider-crs-cni --load_restrictor none > $(TEST_TEMPLATES_TARGET_DIR)/v1alpha4/cluster-template-no-cpem.yaml
+	$(KUSTOMIZE) build $(REPO_ROOT)/test/e2e/data/v1alpha4/cluster-template-kcp-scale-in --load_restrictor none > $(TEST_TEMPLATES_TARGET_DIR)/v1alpha4/cluster-template-kcp-scale-in.yaml
+	$(KUSTOMIZE) build $(REPO_ROOT)/test/e2e/data/v1alpha4/cluster-template-node-drain --load_restrictor none > $(TEST_TEMPLATES_TARGET_DIR)/v1alpha4/cluster-template-node-drain.yaml
+	$(KUSTOMIZE) build $(REPO_ROOT)/test/e2e/data/v1alpha4/cluster-template-md-remediation --load_restrictor none > $(TEST_TEMPLATES_TARGET_DIR)/v1alpha4/cluster-template-md-remediation.yaml
+	$(KUSTOMIZE) build $(REPO_ROOT)/test/e2e/data/v1alpha4/cluster-template-kcp-remediation --load_restrictor none > $(TEST_TEMPLATES_TARGET_DIR)/v1alpha4/cluster-template-kcp-remediation.yaml
 
 ## --------------------------------------
 ## Binaries
@@ -307,11 +305,19 @@ lint: $(GOLANGCI_LINT) ## Lint codebase
 .PHONY: modules
 modules: ## Runs go mod to ensure proper vendoring.
 	go mod tidy
+	cd test/e2e; go mod tidy
 
 .PHONY: generate
 generate: ## Generate code
 	$(MAKE) generate-go
 	$(MAKE) generate-manifests
+	$(MAKE) generate-templates
+
+.PHONY: generate-templates
+generate-templates: $(KUSTOMIZE) ## Generate cluster templates
+	$(KUSTOMIZE) build templates/no-cloud-provider --load_restrictor none > templates/cluster-template-no-cloud-provider.yaml
+	$(KUSTOMIZE) build templates/legacy --load_restrictor none > templates/cluster-template-legacy.yaml
+	$(KUSTOMIZE) build templates/crs-cni-cpem --load_restrictor none > templates/cluster-template.yaml
 
 .PHONY: generate-go
 generate-go: $(CONTROLLER_GEN) $(CONVERSION_GEN) ## Runs Go related generate targets
@@ -436,9 +442,10 @@ release-helper-windows-%:
 release-metadata: $(RELEASE_DIR)
 	cp metadata.yaml $(RELEASE_DIR)/metadata.yaml
 
+# TODO: envsubst to include CNI and/or CPEM resources
 .PHONY: release-templates
 release-templates: $(RELEASE_DIR)
-	cp templates/cluster-template* $(RELEASE_DIR)/
+	cp templates/cluster-template*.yaml $(RELEASE_DIR)/
 
 ## --------------------------------------
 ## Cleanup / Verification

diff --git a/cmd/helper/upgrade/cloudprovider/upgrader/upgrader.go b/cmd/helper/upgrade/cloudprovider/upgrader/upgrader.go
@@ -79,10 +79,6 @@ func New(ctx context.Context, config *base.ToolConfig) (*Upgrader, error) {
 			name:   "RemoveDeprecatedManager",
 			method: removeCCMDeployment,
 		},
-		{
-			name:   "RemoveOldCredentialsSecret",
-			method: removeOldCCMSecret,
-		},
 		{
 			name:   "InstallNewManager",
 			method: installCPEM,
@@ -294,40 +290,6 @@ func installCPEM(ctx context.Context, logger logr.Logger, u *base.Tool, c *clust
 	return nil
 }
 
-func removeOldCCMSecret(ctx context.Context, logger logr.Logger, u *base.Tool, c *clusterv1.Cluster) error {
-	stdout := u.GetBufferFor(c)
-	ccmSecretKey := client.ObjectKey{Namespace: metav1.NamespaceSystem, Name: oldSecretName}
-	csiStatefulSet := new(appsv1.StatefulSet)
-	csiKey := client.ObjectKey{Namespace: metav1.NamespaceSystem, Name: csiStatefulSetName}
-	secretName := fmt.Sprintf("%s/%s", ccmSecretKey.Namespace, ccmSecretKey.Name)
-
-	err := u.WorkloadGet(ctx, c, csiKey, csiStatefulSet)
-
-	switch {
-	case err != nil && !apierrors.IsNotFound(err):
-		return err
-	case err == nil:
-		logger.Info(base.SkipPrefix+"removal of secret because Packet CSI is deployed", "name", secretName)
-		fmt.Fprintf(stdout, "%sremoval of Secret %s because Packet CSI is deployed", base.SkipPrefix, secretName)
-
-		return nil
-	}
-
-	ccmSecret := new(corev1.Secret)
-	if err := u.WorkloadGet(ctx, c, ccmSecretKey, ccmSecret); err != nil {
-		if apierrors.IsNotFound(err) {
-			logger.Info("Secret already removed", "name", secretName)
-			fmt.Fprintf(stdout, "%sSecret %s already deleted\n", base.NoOpPrefix, secretName)
-
-			return nil
-		}
-
-		return err
-	}
-
-	return u.WorkloadDelete(ctx, logger, c, ccmSecret)
-}
-
 func removeCCMDeployment(ctx context.Context, logger logr.Logger, u *base.Tool, c *clusterv1.Cluster) error {
 	stdout := u.GetBufferFor(c)
 	ccmDeployment := new(appsv1.Deployment)

diff --git a/cmd/helper/upgrade/cloudprovider/upgrader/upgrader_internal_test.go b/cmd/helper/upgrade/cloudprovider/upgrader/upgrader_internal_test.go
@@ -109,115 +109,6 @@ func TestUpgrader_removeCCMDeploymentDry(t *testing.T) {
 	g.Expect(actualOldDeployment).To(testutils.BeDerivativeOf(oldDeployment))
 }
 
-func TestUpgrader_removeCCMSecret(t *testing.T) {
-	t.Parallel()
-	g := NewWithT(t)
-	ctx := context.TODO()
-	oldSecret := &corev1.Secret{ //nolint:exhaustivestruct
-		ObjectMeta: metav1.ObjectMeta{ //nolint:exhaustivestruct
-			Namespace: metav1.NamespaceSystem,
-			Name:      oldSecretName,
-		},
-		Data: map[string][]byte{
-			"for": []byte("old-ccm"),
-		},
-	}
-
-	clusterWithoutCCMSecret := testutils.GenerateCluster("", "withoutCCMSecret")
-	clusterWithCCMSecret := testutils.GenerateCluster("", "withCCMSecret")
-	clusterWithCCMSecretAndCSI := testutils.GenerateCluster("", "withCCMSecretAndCSI")
-	workloadResources := map[client.ObjectKey][]client.Object{
-		{Namespace: clusterWithCCMSecret.Namespace, Name: clusterWithCCMSecret.Name}: {
-			oldSecret.DeepCopy(),
-		},
-		{Namespace: clusterWithCCMSecretAndCSI.Namespace, Name: clusterWithCCMSecretAndCSI.Name}: {
-			oldSecret.DeepCopy(),
-			testutils.GenerateStatefulSet(metav1.NamespaceSystem, csiStatefulSetName, "test"),
-		},
-	}
-
-	fakeEnv := testutils.NewFakeEnv(ctx, t, workloadResources, clusterWithoutCCMSecret,
-		clusterWithCCMSecret, clusterWithCCMSecretAndCSI)
-	toolConfig := &base.ToolConfig{ //nolint:exhaustivestruct
-		MgmtClient:           fakeEnv.MgmtClient,
-		WorkloadClientGetter: fakeEnv.WorkloadClientGetter,
-		Logger:               klogr.New(),
-	}
-	u, err := New(ctx, toolConfig)
-	g.Expect(err).NotTo(HaveOccurred())
-
-	preRemoveOutput := u.GetOutputFor(clusterWithoutCCMSecret)
-	g.Expect(removeOldCCMSecret(ctx, toolConfig.Logger, u.Tool, clusterWithoutCCMSecret)).To(Succeed())
-	postRemoveOutput := u.GetOutputFor(clusterWithoutCCMSecret)
-	testutils.VerifySuccessOutputUnchanged(t, strings.TrimPrefix(postRemoveOutput, preRemoveOutput))
-
-	oldSecretKey := client.ObjectKey{Namespace: metav1.NamespaceSystem, Name: oldSecretName}
-	g.Expect(u.WorkloadGet(ctx, clusterWithoutCCMSecret, oldSecretKey, new(corev1.Secret))).
-		To(MatchError(ContainSubstring("not found")))
-
-	preRemoveOutput = u.GetOutputFor(clusterWithCCMSecret)
-	g.Expect(removeOldCCMSecret(ctx, toolConfig.Logger, u.Tool, clusterWithCCMSecret)).To(Succeed())
-	postRemoveOutput = u.GetOutputFor(clusterWithCCMSecret)
-	testutils.VerifySuccessOutputChanged(t, strings.TrimPrefix(postRemoveOutput, preRemoveOutput))
-
-	g.Expect(u.WorkloadGet(ctx, clusterWithCCMSecret, oldSecretKey, new(corev1.Secret))).
-		To(MatchError(ContainSubstring("not found")))
-
-	preRemoveOutput = u.GetOutputFor(clusterWithCCMSecretAndCSI)
-	g.Expect(removeOldCCMSecret(ctx, toolConfig.Logger, u.Tool, clusterWithCCMSecretAndCSI)).To(Succeed())
-	postRemoveOutput = u.GetOutputFor(clusterWithCCMSecretAndCSI)
-	testutils.VerifySuccessOutputSkipped(t, strings.TrimPrefix(postRemoveOutput, preRemoveOutput))
-
-	actualSecret := new(corev1.Secret)
-	g.Expect(u.WorkloadGet(ctx, clusterWithCCMSecretAndCSI, oldSecretKey, actualSecret)).To(Succeed())
-	g.Expect(actualSecret).To(testutils.BeDerivativeOf(oldSecret))
-}
-
-func TestUpgrader_removeOldCCMSecretDry(t *testing.T) {
-	t.Parallel()
-	g := NewWithT(t)
-	ctx := context.TODO()
-	cluster := testutils.GenerateCluster("", "")
-
-	oldSecret := &corev1.Secret{ //nolint:exhaustivestruct
-		ObjectMeta: metav1.ObjectMeta{ //nolint:exhaustivestruct
-			Namespace: metav1.NamespaceSystem,
-			Name:      oldSecretName,
-		},
-		Data: map[string][]byte{
-			"for": []byte("old-ccm"),
-		},
-	}
-
-	workloadResources := map[client.ObjectKey][]client.Object{
-		{Namespace: cluster.Namespace, Name: cluster.Name}: {
-			oldSecret.DeepCopy(),
-		},
-	}
-
-	testEnv := testutils.NewTestEnv(ctx, t, workloadResources, cluster)
-	toolConfig := &base.ToolConfig{ //nolint:exhaustivestruct
-		DryRun:               true,
-		RestConfig:           testEnv.RestConfig,
-		WorkloadClientGetter: testEnv.WorkloadClientGetter,
-		Logger:               klogr.New(),
-	}
-	u, err := New(ctx, toolConfig)
-	g.Expect(err).NotTo(HaveOccurred())
-
-	// Test Dry Run
-	preDryRunOutput := u.GetOutputFor(cluster)
-	g.Expect(removeOldCCMSecret(ctx, toolConfig.Logger, u.Tool, cluster)).To(Succeed())
-	postDryRunOutput := u.GetOutputFor(cluster)
-	testutils.VerifySuccessOutputDryRun(t, strings.TrimPrefix(postDryRunOutput, preDryRunOutput))
-
-	// Ensure the oldSecret still exists
-	oldSecretKey := client.ObjectKey{Namespace: metav1.NamespaceSystem, Name: oldSecretName}
-	actualOldSecret := new(corev1.Secret)
-	g.Expect(u.WorkloadGet(ctx, cluster, oldSecretKey, actualOldSecret)).To(Succeed())
-	g.Expect(actualOldSecret).To(testutils.BeDerivativeOf(oldSecret))
-}
-
 func TestUpgrader_migrateSecret(t *testing.T) {
 	t.Parallel()
 	g := NewWithT(t)

diff --git a/controllers/packetmachine_controller.go b/controllers/packetmachine_controller.go
@@ -316,6 +316,7 @@ func (r *PacketMachineReconciler) reconcile(ctx context.Context, machineScope *s
 			ExtraTags:    packet.DefaultCreateTags(machineScope.Namespace(), machineScope.Machine.Name, machineScope.Cluster.Name),
 		}
 
+		// TODO: see if this can be removed with kube-vip in place
 		// when the node is a control plan we should check if the elastic ip
 		// for this cluster is not assigned. If it is free we can prepare the
 		// current node to use it.

diff --git a/go.mod b/go.mod
@@ -7,19 +7,19 @@ require (
 	github.com/charmbracelet/bubbles v0.8.0
 	github.com/charmbracelet/bubbletea v0.14.1
 	github.com/charmbracelet/lipgloss v0.3.0
+	github.com/containerd/console v1.0.2 // indirect
 	github.com/docker/distribution v2.7.1+incompatible
 	github.com/go-logr/logr v0.4.0
 	github.com/google/go-cmp v0.5.6
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/mattn/go-isatty v0.0.13
 	github.com/muesli/reflow v0.2.1-0.20210115123740-9e1d0d53df68
-	github.com/onsi/ginkgo v1.16.4
 	github.com/onsi/gomega v1.16.0
 	github.com/packethost/packngo v0.19.0
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/pflag v1.0.5
-	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
+	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect
 	k8s.io/api v0.21.3
 	k8s.io/apiextensions-apiserver v0.21.3
 	k8s.io/apimachinery v0.21.3
@@ -28,9 +28,6 @@ require (
 	k8s.io/klog/v2 v2.10.0
 	k8s.io/utils v0.0.0-20210722164352-7f3ee0f31471
 	sigs.k8s.io/cluster-api v0.4.2
-	sigs.k8s.io/cluster-api/test v0.4.2
 	sigs.k8s.io/controller-runtime v0.9.6
 	sigs.k8s.io/yaml v1.2.0
 )
-
-replace sigs.k8s.io/cluster-api => sigs.k8s.io/cluster-api v0.4.2