cli/cmd: upgrade controlplane as part of install process

This commit adds controlplane upgrade functionality to 'cluster install'
command. If user runs this command on existing cluster, all controlplane
helm releases will be upgraded using charts and values.yaml generated by
Terraform.

If some of controlplane release has been uninstalled by the user, it
will be reinstalled to ensure consitency.

The upgrade process is atomic, which means that if something goes wrong
(e.g. new pod will not become ready), then the upgrade will be rolled
back.

The upgrade process is done in order recommended by upstream:
https://kubernetes.io/docs/setup/release/version-skew-policy/#supported-component-upgrade-order

Currently, 'kubelet' is not being upgraded, because of bug in runc on
Flatcar Linux. See #110 for
more details. For testing, --upgrade-kubelets can be used, but it's
experimental and not by default at the moment.

Signed-off-by: Mateusz Gozdek <mateusz@kinvolk.io>

cli/cmd/cluster-install.go

@@ -27,8 +27,9 @@ import (
 )
 
 var (
-	quiet          bool
-	skipComponents bool
+	quiet           bool
+	skipComponents  bool
+	upgradeKubelets bool
 )
 
 var clusterInstallCmd = &cobra.Command{
@@ -43,6 +44,7 @@ func init() {
 	pf.BoolVarP(&confirm, "confirm", "", false, "Upgrade cluster without asking for confirmation")
 	pf.BoolVarP(&quiet, "quiet", "q", false, "Suppress the output from Terraform")
 	pf.BoolVarP(&skipComponents, "skip-components", "", false, "Skip component installation")
+	pf.BoolVarP(&upgradeKubelets, "upgrade-kubelets", "", false, "Experimentall upgrade self-hosted kubelets")
 }
 
 func runClusterInstall(cmd *cobra.Command, args []string) {
@@ -53,7 +55,8 @@ func runClusterInstall(cmd *cobra.Command, args []string) {
 
 	ex, p, lokoConfig, assetDir := initialize(ctxLogger)
 
-	if clusterExists(ctxLogger, ex) && !confirm {
+	exists := clusterExists(ctxLogger, ex)
+	if exists && !confirm {
 		// TODO: We could plan to a file and use it when installing.
 		if err := ex.Plan(); err != nil {
 			ctxLogger.Fatalf("Failed to reconsile cluster state: %v", err)
@@ -77,6 +80,33 @@ func runClusterInstall(cmd *cobra.Command, args []string) {
 		ctxLogger.Fatalf("Verify cluster installation: %v", err)
 	}
 
+	// Do controlplane upgrades only if cluster already exists.
+	if exists {
+		fmt.Printf("\nEnsuring that cluster controlplane is up to date.\n")
+
+		networking := ""
+		if err := ex.Output("networking", &networking); err != nil {
+			ctxLogger.Fatalf("Failed checking which networking solution is in use: %v", err)
+		}
+
+		cu := controlplaneUpdater{
+			kubeconfigPath: kubeconfigPath,
+			assetDir:       assetDir,
+			ctxLogger:      *ctxLogger,
+			ex:             *ex,
+		}
+
+		releases := []string{"pod-checkpointer", "kube-apiserver", "kubernetes", networking}
+
+		if upgradeKubelets {
+			releases = append(releases, "kubelet")
+		}
+
+		for _, c := range releases {
+			cu.upgradeComponent(c)
+		}
+	}
+
 	if skipComponents {
 		return
 	}

cli/cmd/cluster.go

@@ -15,12 +15,20 @@
 package cmd
 
 import (
+	"fmt"
+	"path/filepath"
+
 	"github.com/mitchellh/go-homedir"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chart"
+	"helm.sh/helm/v3/pkg/chart/loader"
+	"sigs.k8s.io/yaml"
 
 	"github.com/kinvolk/lokomotive/pkg/backend"
 	"github.com/kinvolk/lokomotive/pkg/backend/local"
+	"github.com/kinvolk/lokomotive/pkg/components/util"
 	"github.com/kinvolk/lokomotive/pkg/config"
 	"github.com/kinvolk/lokomotive/pkg/platform"
 	"github.com/kinvolk/lokomotive/pkg/terraform"
@@ -138,3 +146,95 @@ func clusterExists(ctxLogger *logrus.Entry, ex *terraform.Executor) bool {
 
 	return len(o) != 0
 }
+
+type controlplaneUpdater struct {
+	kubeconfigPath string
+	assetDir       string
+	ctxLogger      logrus.Entry
+	ex             terraform.Executor
+}
+
+func (c controlplaneUpdater) getControlplaneChart(name string) (*chart.Chart, error) {
+	helmChart, err := loader.Load(filepath.Join(c.assetDir, "/lokomotive-kubernetes/bootkube/resources/charts", name))
+	if err != nil {
+		return nil, fmt.Errorf("loading chart from assets failed: %w", err)
+	}
+
+	if err := helmChart.Validate(); err != nil {
+		return nil, fmt.Errorf("chart is invalid: %w", err)
+	}
+
+	return helmChart, nil
+}
+
+func (c controlplaneUpdater) getControlplaneValues(name string) (map[string]interface{}, error) {
+	valuesRaw := ""
+	if err := c.ex.Output(fmt.Sprintf("%s_values", name), &valuesRaw); err != nil {
+		return nil, fmt.Errorf("failed to get kubernetes values.yaml from Terraform: %w", err)
+	}
+
+	values := map[string]interface{}{}
+	if err := yaml.Unmarshal([]byte(valuesRaw), &values); err != nil {
+		return nil, fmt.Errorf("failed to parse values.yaml for kubernetes: %w", err)
+	}
+
+	return values, nil
+}
+
+func (c controlplaneUpdater) upgradeComponent(component string) {
+	ctxLogger := c.ctxLogger.WithFields(logrus.Fields{
+		"action":    "controlplane-upgrade",
+		"component": component,
+	})
+
+	actionConfig, err := util.HelmActionConfig("kube-system", c.kubeconfigPath)
+	if err != nil {
+		ctxLogger.Fatalf("Failed initializing helm: %v", err)
+	}
+
+	helmChart, err := c.getControlplaneChart(component)
+	if err != nil {
+		ctxLogger.Fatalf("Loading chart from assets failed: %v", err)
+	}
+
+	values, err := c.getControlplaneValues(component)
+	if err != nil {
+		ctxLogger.Fatalf("Failed to get kubernetes values.yaml from Terraform: %v", err)
+	}
+
+	exists, err := util.ReleaseExists(*actionConfig, component)
+	if err != nil {
+		ctxLogger.Fatalf("Failed checking if controlplane component is installed: %v", err)
+	}
+
+	if !exists {
+		fmt.Printf("Controlplane component '%s' is missing, reinstalling...", component)
+
+		install := action.NewInstall(actionConfig)
+		install.ReleaseName = component
+		install.Namespace = "kube-system"
+		install.Atomic = true
+
+		if _, err := install.Run(helmChart, map[string]interface{}{}); err != nil {
+			fmt.Println("Failed!")
+
+			ctxLogger.Fatalf("Installing controlplane component failed: %v", err)
+		}
+
+		fmt.Println("Done.")
+	}
+
+	update := action.NewUpgrade(actionConfig)
+
+	update.Atomic = true
+
+	fmt.Printf("Ensuring controlplane component '%s' is up to date... ", component)
+
+	if _, err := update.Run(component, helmChart, values); err != nil {
+		fmt.Println("Failed!")
+
+		ctxLogger.Fatalf("Updating chart failed: %v", err)
+	}
+
+	fmt.Println("Done.")
+}