diff --git a/cli/cmd/cluster-install.go b/cli/cmd/cluster-install.go index 18b88d436..903d308cd 100644 --- a/cli/cmd/cluster-install.go +++ b/cli/cmd/cluster-install.go @@ -27,8 +27,9 @@ import ( ) var ( - quiet bool - skipComponents bool + quiet bool //nolintgo:checknoglobals + skipComponents bool //nolintgo:checknoglobals + upgradeKubelets bool //nolintgo:checknoglobals ) var clusterInstallCmd = &cobra.Command{ @@ -43,6 +44,7 @@ func init() { pf.BoolVarP(&confirm, "confirm", "", false, "Upgrade cluster without asking for confirmation") pf.BoolVarP(&quiet, "quiet", "q", false, "Suppress the output from Terraform") pf.BoolVarP(&skipComponents, "skip-components", "", false, "Skip component installation") + pf.BoolVarP(&upgradeKubelets, "upgrade-kubelets", "", false, "Experimentally upgrade self-hosted kubelets") } func runClusterInstall(cmd *cobra.Command, args []string) { @@ -53,7 +55,8 @@ func runClusterInstall(cmd *cobra.Command, args []string) { ex, p, lokoConfig, assetDir := initialize(ctxLogger) - if clusterExists(ctxLogger, ex) && !confirm { + exists := clusterExists(ctxLogger, ex) + if exists && !confirm { // TODO: We could plan to a file and use it when installing. if err := ex.Plan(); err != nil { ctxLogger.Fatalf("Failed to reconsile cluster state: %v", err) @@ -77,6 +80,33 @@ func runClusterInstall(cmd *cobra.Command, args []string) { ctxLogger.Fatalf("Verify cluster installation: %v", err) } + // Do controlplane upgrades only if cluster already exists. + if exists { + fmt.Printf("\nEnsuring that cluster controlplane is up to date.\n") + + networking := "" + if err := ex.Output("networking", &networking); err != nil { + ctxLogger.Fatalf("Failed checking which networking solution is in use: %v", err) + } + + cu := controlplaneUpdater{ + kubeconfigPath: kubeconfigPath, + assetDir: assetDir, + ctxLogger: *ctxLogger, + ex: *ex, + } + + releases := []string{"pod-checkpointer", "kube-apiserver", "kubernetes", networking} + + if upgradeKubelets { + releases = append(releases, "kubelet") + } + + for _, c := range releases { + cu.upgradeComponent(c) + } + } + if skipComponents { return } diff --git a/cli/cmd/cluster.go b/cli/cmd/cluster.go index 61fb291a8..bc36ccaf7 100644 --- a/cli/cmd/cluster.go +++ b/cli/cmd/cluster.go @@ -15,12 +15,20 @@ package cmd import ( + "fmt" + "path/filepath" + "github.com/mitchellh/go-homedir" "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "helm.sh/helm/v3/pkg/action" + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chart/loader" + "sigs.k8s.io/yaml" "github.com/kinvolk/lokomotive/pkg/backend" "github.com/kinvolk/lokomotive/pkg/backend/local" + "github.com/kinvolk/lokomotive/pkg/components/util" "github.com/kinvolk/lokomotive/pkg/config" "github.com/kinvolk/lokomotive/pkg/platform" "github.com/kinvolk/lokomotive/pkg/terraform" @@ -138,3 +146,95 @@ func clusterExists(ctxLogger *logrus.Entry, ex *terraform.Executor) bool { return len(o) != 0 } + +type controlplaneUpdater struct { + kubeconfigPath string + assetDir string + ctxLogger logrus.Entry + ex terraform.Executor +} + +func (c controlplaneUpdater) getControlplaneChart(name string) (*chart.Chart, error) { + helmChart, err := loader.Load(filepath.Join(c.assetDir, "/lokomotive-kubernetes/bootkube/resources/charts", name)) + if err != nil { + return nil, fmt.Errorf("loading chart from assets failed: %w", err) + } + + if err := helmChart.Validate(); err != nil { + return nil, fmt.Errorf("chart is invalid: %w", err) + } + + return helmChart, nil +} + +func (c controlplaneUpdater) getControlplaneValues(name string) (map[string]interface{}, error) { + valuesRaw := "" + if err := c.ex.Output(fmt.Sprintf("%s_values", name), &valuesRaw); err != nil { + return nil, fmt.Errorf("failed to get kubernetes values.yaml from Terraform: %w", err) + } + + values := map[string]interface{}{} + if err := yaml.Unmarshal([]byte(valuesRaw), &values); err != nil { + return nil, fmt.Errorf("failed to parse values.yaml for kubernetes: %w", err) + } + + return values, nil +} + +func (c controlplaneUpdater) upgradeComponent(component string) { + ctxLogger := c.ctxLogger.WithFields(logrus.Fields{ + "action": "controlplane-upgrade", + "component": component, + }) + + actionConfig, err := util.HelmActionConfig("kube-system", c.kubeconfigPath) + if err != nil { + ctxLogger.Fatalf("Failed initializing helm: %v", err) + } + + helmChart, err := c.getControlplaneChart(component) + if err != nil { + ctxLogger.Fatalf("Loading chart from assets failed: %v", err) + } + + values, err := c.getControlplaneValues(component) + if err != nil { + ctxLogger.Fatalf("Failed to get kubernetes values.yaml from Terraform: %v", err) + } + + exists, err := util.ReleaseExists(*actionConfig, component) + if err != nil { + ctxLogger.Fatalf("Failed checking if controlplane component is installed: %v", err) + } + + if !exists { + fmt.Printf("Controlplane component '%s' is missing, reinstalling...", component) + + install := action.NewInstall(actionConfig) + install.ReleaseName = component + install.Namespace = "kube-system" + install.Atomic = true + + if _, err := install.Run(helmChart, map[string]interface{}{}); err != nil { + fmt.Println("Failed!") + + ctxLogger.Fatalf("Installing controlplane component failed: %v", err) + } + + fmt.Println("Done.") + } + + update := action.NewUpgrade(actionConfig) + + update.Atomic = true + + fmt.Printf("Ensuring controlplane component '%s' is up to date... ", component) + + if _, err := update.Run(component, helmChart, values); err != nil { + fmt.Println("Failed!") + + ctxLogger.Fatalf("Updating chart failed: %v", err) + } + + fmt.Println("Done.") +}