If upgrading CRD has different versions than the ones stored on the s…

…erver, remove the previously stored versions as stored versions

Makefile

@@ -1,4 +1,4 @@
-VERSION ?= 0.3.0
+VERSION ?= 0.7.0
 
 COMMIT := $(shell git rev-parse --short HEAD)
 DATE := $(shell date +%Y%m%d)
@@ -14,7 +14,7 @@ SHELL = /usr/bin/env bash -o pipefail
 .SHELLFLAGS = -ec
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
-ENVTEST_K8S_VERSION = 1.28.x
+ENVTEST_K8S_VERSION = 1.31.x
 
 GO_FLAGS ?= -v
 
@@ -83,7 +83,7 @@ $(LOCALBIN):
 GOLANGCI_LINT ?= $(LOCALBIN)/golangci-lint
 ENVTEST ?= $(LOCALBIN)/setup-envtest
 
-GOLINT_VERSION ?= 1.56.2
+GOLINT_VERSION ?= 1.61.0
 
 .PHONY: envtest
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.

pkg/helmutil/crds.go

@@ -8,12 +8,15 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"github.com/charmbracelet/log"
 	"github.com/pkg/errors"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	deser "k8s.io/apimachinery/pkg/runtime/serializer/yaml"
 	k8syaml "k8s.io/apimachinery/pkg/util/yaml"
 
@@ -48,7 +51,7 @@ func NewUpgrader(c client.Client, repoName, repoURL, chartName string, subCharts
 func (u *Upgrader) Upgrade(ctx context.Context, chartVersion string) ([]unstructured.Unstructured, error) {
 	log.SetLevel(log.DebugLevel)
 	log.Info("Processing request to upgrade project CustomResourceDefinitions", "repoName", u.repoName, "chartName", u.chartName, "chartVersion", chartVersion)
-	chartDir, err := GetChartTargetDir(u.repoName, u.chartName)
+	chartDir, err := GetChartTargetDir(u.repoName, u.chartName, chartVersion)
 	if err != nil {
 		return nil, err
 	}
@@ -102,6 +105,51 @@ func (u *Upgrader) Upgrade(ctx context.Context, chartVersion string) ([]unstruct
 		} else {
 			log.Debug("Updating CustomResourceDefinition", "name", obj.GetName())
 			obj.SetResourceVersion(existingCrd.GetResourceVersion())
+
+			// TODO We need to check which versions we have available here before updating
+			unstructured := obj.UnstructuredContent()
+			var definition apiextensionsv1.CustomResourceDefinition
+			if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructured, &definition); err != nil {
+				return nil, errors.Wrapf(err, "failed to convert unstructured to CustomResourceDefinition %s", obj.GetName())
+			}
+
+			updatedVersions := make([]string, 0, len(definition.Spec.Versions))
+			for _, version := range definition.Spec.Versions {
+				updatedVersions = append(updatedVersions, version.Name)
+			}
+			log.Debug("Read CustomResourceDefinition versions", "name", obj.GetName(), "versions", updatedVersions)
+
+			existing := existingCrd.UnstructuredContent()
+			var existingDefinition apiextensionsv1.CustomResourceDefinition
+			if err := runtime.DefaultUnstructuredConverter.FromUnstructured(existing, &existingDefinition); err != nil {
+				return nil, errors.Wrapf(err, "failed to convert unstructured to CustomResourceDefinition %s", obj.GetName())
+			}
+
+			storedVersions := existingDefinition.Status.StoredVersions
+
+			if !slices.Equal(storedVersions, updatedVersions) {
+
+				// Check if storedVersion has any versions that are not in updatedVersions
+				// If so, we need to remove them from the storedVersions
+				removed := false
+				for _, storedVersion := range storedVersions {
+					if !slices.Contains(updatedVersions, storedVersion) {
+						log.Debug("Removing CustomResourceDefinition version", "name", obj.GetName(), "version", storedVersion)
+						// storedVersions = slices.DeleteFunc(storedVersions, func(e string) bool { return e == storedVersion })
+						removed = true
+					}
+				}
+
+				if removed {
+					log.Debug("Updating CustomResourceDefinition versions", "name", obj.GetName(), "storedVersions", storedVersions, "updatedVersions", updatedVersions)
+					existingDefinition.Status.StoredVersions = updatedVersions
+					if err := u.client.Status().Update(ctx, &existingDefinition); err != nil {
+						return nil, errors.Wrapf(err, "failed to update CRD storedVersions %s", obj.GetName())
+					}
+					obj.SetResourceVersion(existingDefinition.GetResourceVersion())
+				}
+			}
+
 			if err = u.client.Update(ctx, &obj); err != nil {
 				return nil, errors.Wrapf(err, "failed to update CRD %s", obj.GetName())
 			}

pkg/helmutil/crds_test.go

@@ -2,12 +2,17 @@ package helmutil_test
 
 import (
 	"context"
+	"fmt"
+	"io"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
 	"github.com/k8ssandra/k8ssandra-client/pkg/helmutil"
+	"github.com/k8ssandra/k8ssandra-client/pkg/util"
+	"github.com/pkg/errors"
 	"github.com/stretchr/testify/require"
 	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -25,7 +30,7 @@ func TestUpgradingCRDs(t *testing.T) {
 	for _, chartName := range chartNames {
 		namespace := env.CreateNamespace(t)
 		kubeClient := env.GetClientInNamespace(namespace)
-		require.NoError(cleanCache("k8ssandra", chartName))
+		require.NoError(cleanCache("k8ssandra", chartName, "0.42.0"))
 
 		// creating new upgrader
 		u, err := helmutil.NewUpgrader(kubeClient, helmutil.K8ssandraRepoName, helmutil.StableK8ssandraRepoURL, chartName, []string{})
@@ -59,7 +64,7 @@ func TestUpgradingCRDs(t *testing.T) {
 		require.False(strings.HasPrefix(descRunsAsCassandra, "DEPRECATED"))
 
 		// Upgrading to 0.46.1
-		require.NoError(cleanCache("k8ssandra", chartName))
+		require.NoError(cleanCache("k8ssandra", chartName, "0.46.1"))
 		_, err = u.Upgrade(context.TODO(), "0.46.1")
 		require.NoError(err)
 
@@ -77,11 +82,160 @@ func TestUpgradingCRDs(t *testing.T) {
 	}
 }
 
-func cleanCache(repoName, chartName string) error {
-	chartDir, err := helmutil.GetChartTargetDir(repoName, chartName)
+func cleanCache(repoName, chartName, version string) error {
+	chartDir, err := helmutil.GetChartTargetDir(repoName, chartName, version)
 	if err != nil {
 		return err
 	}
 
 	return os.RemoveAll(chartDir)
 }
+
+func TestUpgradingStoredVersions(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test in short mode.")
+	}
+
+	require := require.New(t)
+	chartName := "test-chart"
+	namespace := env.CreateNamespace(t)
+	kubeClient := env.GetClientInNamespace(namespace)
+	require.NoError(cleanCache("k8ssandra", chartName, "0.1.0"))
+	require.NoError(cleanCache("k8ssandra", chartName, "0.2.0"))
+	require.NoError(cleanCache("k8ssandra", chartName, "0.3.0"))
+
+	// Copy testfiles
+	chartDir, err := helmutil.GetChartTargetDir(helmutil.K8ssandraRepoName, chartName, "0.1.0")
+	require.NoError(err)
+
+	crdDir := filepath.Join(chartDir, "crds")
+	_, err = util.CreateIfNotExistsDir(crdDir)
+	require.NoError(err)
+	crdSrc := filepath.Join("..", "..", "testfiles", "crd-upgrader", "multiversion-clientconfig-mockup-v1alpha1.yaml")
+	require.NoError(copyFile(crdSrc, filepath.Join(crdDir, "clientconfig.yaml")))
+
+	chartDir, err = helmutil.GetChartTargetDir(helmutil.K8ssandraRepoName, chartName, "0.2.0")
+	require.NoError(err)
+
+	crdDir = filepath.Join(chartDir, "crds")
+	_, err = util.CreateIfNotExistsDir(crdDir)
+	require.NoError(err)
+	crdSrc = filepath.Join("..", "..", "testfiles", "crd-upgrader", "multiversion-clientconfig-mockup-both.yaml")
+	require.NoError(copyFile(crdSrc, filepath.Join(crdDir, "clientconfig.yaml")))
+
+	chartDir, err = helmutil.GetChartTargetDir(helmutil.K8ssandraRepoName, chartName, "0.3.0")
+	require.NoError(err)
+
+	crdDir = filepath.Join(chartDir, "crds")
+	_, err = util.CreateIfNotExistsDir(crdDir)
+	require.NoError(err)
+	crdSrc = filepath.Join("..", "..", "testfiles", "crd-upgrader", "multiversion-clientconfig-mockup-v1beta1.yaml")
+	require.NoError(copyFile(crdSrc, filepath.Join(crdDir, "clientconfig.yaml")))
+
+	testOptions := envtest.CRDInstallOptions{
+		PollInterval: 100 * time.Millisecond,
+		MaxTime:      10 * time.Second,
+	}
+
+	// creating new upgrader
+	u, err := helmutil.NewUpgrader(kubeClient, helmutil.K8ssandraRepoName, helmutil.StableK8ssandraRepoURL, chartName, []string{})
+	require.NoError(err)
+
+	crds, err := u.Upgrade(context.TODO(), "0.1.0")
+	require.NoError(err)
+
+	targetCrd := &apiextensions.CustomResourceDefinition{}
+	objs := []*apiextensions.CustomResourceDefinition{}
+	for _, crd := range crds {
+		err = runtime.DefaultUnstructuredConverter.FromUnstructured(crd.UnstructuredContent(), targetCrd)
+		require.NoError(err)
+		objs = append(objs, targetCrd)
+	}
+
+	require.NotEmpty(objs)
+	require.NotEmpty(targetCrd.GetName())
+	require.NoError(envtest.WaitForCRDs(env.RestConfig(), objs, testOptions))
+	require.NoError(kubeClient.Get(context.TODO(), client.ObjectKey{Name: targetCrd.GetName()}, targetCrd))
+
+	require.Equal([]string{"v1alpha1"}, targetCrd.Status.StoredVersions)
+
+	// Upgrade to 0.2.0
+
+	crds, err = u.Upgrade(context.TODO(), "0.2.0")
+	require.NoError(err)
+	for _, crd := range crds {
+		err = runtime.DefaultUnstructuredConverter.FromUnstructured(crd.UnstructuredContent(), targetCrd)
+		require.NoError(err)
+		objs = append(objs, targetCrd)
+	}
+	require.NotEmpty(objs)
+	require.NoError(envtest.WaitForCRDs(env.RestConfig(), objs, testOptions))
+	require.NoError(kubeClient.Get(context.TODO(), client.ObjectKey{Name: targetCrd.GetName()}, targetCrd))
+	require.Equal([]string{"v1alpha1", "v1beta1"}, targetCrd.Status.StoredVersions)
+
+	// Upgrade to 0.3.0
+
+	crds, err = u.Upgrade(context.TODO(), "0.3.0")
+	require.NoError(err)
+	for _, crd := range crds {
+		err = runtime.DefaultUnstructuredConverter.FromUnstructured(crd.UnstructuredContent(), targetCrd)
+		require.NoError(err)
+		objs = append(objs, targetCrd)
+	}
+	require.NotEmpty(objs)
+	require.NoError(envtest.WaitForCRDs(env.RestConfig(), objs, testOptions))
+	require.NoError(kubeClient.Get(context.TODO(), client.ObjectKey{Name: targetCrd.GetName()}, targetCrd))
+	require.Equal([]string{"v1beta1"}, targetCrd.Status.StoredVersions)
+
+	// Sanity check, install 0.2.0 and only update to 0.3.0 (there should be no storedVersion of v1alpha1)
+	require.NoError(kubeClient.Delete(context.TODO(), targetCrd))
+	require.Eventually(func() bool {
+		err = kubeClient.Get(context.TODO(), client.ObjectKey{Name: targetCrd.GetName()}, targetCrd)
+		return err != nil && client.IgnoreNotFound(err) == nil
+	}, time.Second*5, time.Millisecond*100)
+
+	// Install 0.2.0
+
+	crds, err = u.Upgrade(context.TODO(), "0.2.0")
+	require.NoError(err)
+	for _, crd := range crds {
+		err = runtime.DefaultUnstructuredConverter.FromUnstructured(crd.UnstructuredContent(), targetCrd)
+		require.NoError(err)
+		objs = append(objs, targetCrd)
+	}
+	require.NotEmpty(objs)
+	require.NoError(envtest.WaitForCRDs(env.RestConfig(), objs, testOptions))
+	require.NoError(kubeClient.Get(context.TODO(), client.ObjectKey{Name: targetCrd.GetName()}, targetCrd))
+	require.Equal([]string{"v1beta1"}, targetCrd.Status.StoredVersions)
+
+	// Upgrade to 0.3.0
+
+	crds, err = u.Upgrade(context.TODO(), "0.3.0")
+	require.NoError(err)
+	for _, crd := range crds {
+		err = runtime.DefaultUnstructuredConverter.FromUnstructured(crd.UnstructuredContent(), targetCrd)
+		require.NoError(err)
+		objs = append(objs, targetCrd)
+	}
+	require.NotEmpty(objs)
+	require.NoError(envtest.WaitForCRDs(env.RestConfig(), objs, testOptions))
+	require.NoError(kubeClient.Get(context.TODO(), client.ObjectKey{Name: targetCrd.GetName()}, targetCrd))
+	require.Equal([]string{"v1beta1"}, targetCrd.Status.StoredVersions)
+}
+
+func copyFile(source, target string) error {
+	src, err := os.Open(source)
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("failed to open %s", source))
+	}
+	defer src.Close()
+
+	dst, err := os.Create(target)
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("failed to open %s", target))
+	}
+	defer dst.Close()
+
+	_, err = io.Copy(dst, src)
+	return err
+}

pkg/helmutil/fetch.go

@@ -110,12 +110,14 @@ func ExtractChartRelease(saved, repoName, chartName, chartVersion string) (strin
 	return extractDir, nil
 }
 
-func GetChartTargetDir(repoName, chartName string) (string, error) {
+func GetChartTargetDir(repoName, chartName, chartVersion string) (string, error) {
 	extractDir, err := util.GetCacheDir(repoName, chartName)
 	if err != nil {
 		return "", err
 	}
 
+	extractDir = filepath.Join(extractDir, chartVersion)
+
 	return extractDir, err
 }