clusterctl init: wait for deployments to be ready

cmd/clusterctl/client/cluster/installer.go

@@ -17,14 +17,23 @@ limitations under the License.
 package cluster
 
 import (
+	"context"
+	"time"
+
 	"github.com/pkg/errors"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/apimachinery/pkg/util/wait"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha4"
 	clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/client/config"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/client/repository"
+	"sigs.k8s.io/cluster-api/cmd/clusterctl/internal/util"
 	logf "sigs.k8s.io/cluster-api/cmd/clusterctl/log"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // ProviderInstaller defines methods for enforcing consistency rules for provider installation.
@@ -35,7 +44,7 @@ type ProviderInstaller interface {
 	Add(repository.Components)
 
 	// Install performs the installation of the providers ready in the install queue.
-	Install() ([]repository.Components, error)
+	Install(InstallOptions) ([]repository.Components, error)
 
 	// Validate performs steps to validate a management cluster by looking at the current state and the providers in the queue.
 	// The following checks are performed in order to ensure a fully operational cluster:
@@ -47,6 +56,12 @@ type ProviderInstaller interface {
 	Images() []string
 }
 
+// InstallOptions defines the options used to configure installation.
+type InstallOptions struct {
+	WaitProviders       bool
+	WaitProviderTimeout time.Duration
+}
+
 // providerInstaller implements ProviderInstaller.
 type providerInstaller struct {
 	configClient            config.Client
@@ -63,7 +78,7 @@ func (i *providerInstaller) Add(components repository.Components) {
 	i.installQueue = append(i.installQueue, components)
 }
 
-func (i *providerInstaller) Install() ([]repository.Components, error) {
+func (i *providerInstaller) Install(opts InstallOptions) ([]repository.Components, error) {
 	ret := make([]repository.Components, 0, len(i.installQueue))
 	for _, components := range i.installQueue {
 		if err := installComponentsAndUpdateInventory(components, i.providerComponents, i.providerInventory); err != nil {
@@ -72,6 +87,11 @@ func (i *providerInstaller) Install() ([]repository.Components, error) {
 
 		ret = append(ret, components)
 	}
+
+	if err := i.waitForProvidersReady(opts); err != nil {
+		return ret, err
+	}
+
 	return ret, nil
 }
 
@@ -90,6 +110,57 @@ func installComponentsAndUpdateInventory(components repository.Components, provi
 	return providerInventory.Create(inventoryObject)
 }
 
+// waitForProvidersReady waits till the installed components are ready.
+func (i *providerInstaller) waitForProvidersReady(opts InstallOptions) error {
+	// If we dont have to wait for providers to be installed
+	// return early.
+	if !opts.WaitProviders {
+		return nil
+	}
+
+	log := logf.Log
+	log.Info("Waiting for providers to be available...")
+
+	return i.waitManagerDeploymentsReady(opts)
+}
+
+// waitDeploymentsReady waits till the installed deployments are ready.
+func (i *providerInstaller) waitManagerDeploymentsReady(opts InstallOptions) error {
+	for _, components := range i.installQueue {
+		for _, obj := range components.Objs() {
+			if util.IsDeploymentWithManager(obj) {
+				if err := i.waitDeploymentReady(obj, opts.WaitProviderTimeout); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (i *providerInstaller) waitDeploymentReady(deployment unstructured.Unstructured, timeout time.Duration) error {
+	return wait.Poll(100*time.Millisecond, timeout, func() (bool, error) {
+		c, err := i.proxy.NewClient()
+		if err != nil {
+			return false, err
+		}
+		key := client.ObjectKey{
+			Namespace: deployment.GetNamespace(),
+			Name:      deployment.GetName(),
+		}
+		dep := &appsv1.Deployment{}
+		if err := c.Get(context.TODO(), key, dep); err != nil {
+			return false, err
+		}
+		for _, c := range dep.Status.Conditions {
+			if c.Type == appsv1.DeploymentAvailable && c.Status == corev1.ConditionTrue {
+				return true, nil
+			}
+		}
+		return false, nil
+	})
+}
+
 func (i *providerInstaller) Validate() error {
 	// Get the list of providers currently in the cluster.
 	providerList, err := i.providerInventory.List()

cmd/clusterctl/client/init.go

@@ -18,6 +18,7 @@ package client
 
 import (
 	"sort"
+	"time"
 
 	"github.com/pkg/errors"
 	clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
@@ -58,6 +59,12 @@ type InitOptions struct {
 	// LogUsageInstructions instructs the init command to print the usage instructions in case of first run.
 	LogUsageInstructions bool
 
+	// WaitProviders instructs the init command to wait till the providers are installed.
+	WaitProviders bool
+
+	// WaitProviderTimeout sets the timeout per provider wait installation
+	WaitProviderTimeout time.Duration
+
 	// SkipTemplateProcess allows for skipping the call to the template processor, including also variable replacement in the component YAML.
 	// NOTE this works only if the rawYaml is a valid yaml by itself, like e.g when using envsubst/the simple processor.
 	skipTemplateProcess bool
@@ -109,7 +116,11 @@ func (c *clusterctlClient) Init(options InitOptions) ([]Components, error) {
 		return nil, err
 	}
 
-	components, err := installer.Install()
+	installOpts := cluster.InstallOptions{
+		WaitProviders:       options.WaitProviders,
+		WaitProviderTimeout: options.WaitProviderTimeout,
+	}
+	components, err := installer.Install(installOpts)
 	if err != nil {
 		return nil, err
 	}

cmd/clusterctl/cmd/init.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/spf13/cobra"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/client"
@@ -32,6 +33,8 @@ type initOptions struct {
 	infrastructureProviders []string
 	targetNamespace         string
 	listImages              bool
+	waitProviders           bool
+	waitProviderTimeout     int
 }
 
 var initOpts = &initOptions{}
@@ -100,6 +103,10 @@ func init() {
 		"Control plane providers and versions (e.g. kubeadm:v0.3.0) to add to the management cluster. If unspecified, the Kubeadm control plane provider's latest release is used.")
 	initCmd.Flags().StringVar(&initOpts.targetNamespace, "target-namespace", "",
 		"The target namespace where the providers should be deployed. If unspecified, the provider components' default namespace is used.")
+	initCmd.Flags().BoolVar(&initOpts.waitProviders, "wait-providers", false,
+		"Wait for providers to be installed.")
+	initCmd.Flags().IntVar(&initOpts.waitProviderTimeout, "wait-provider-timeout", 5*60,
+		"Wait timeout per provider installation in seconds. This value is ignored if --wait-providers is false")
 
 	// TODO: Move this to a sub-command or similar, it shouldn't really be a flag.
 	initCmd.Flags().BoolVar(&initOpts.listImages, "list-images", false,
@@ -122,6 +129,8 @@ func runInit() error {
 		InfrastructureProviders: initOpts.infrastructureProviders,
 		TargetNamespace:         initOpts.targetNamespace,
 		LogUsageInstructions:    true,
+		WaitProviders:           initOpts.waitProviders,
+		WaitProviderTimeout:     time.Duration(initOpts.waitProviderTimeout) * time.Second,
 	}
 
 	if initOpts.listImages {

cmd/clusterctl/internal/util/objs.go

@@ -21,6 +21,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/internal/scheme"
 )
 
@@ -173,3 +174,20 @@ func fixContainersImage(containers []corev1.Container, alterImageFunc func(image
 	}
 	return nil
 }
+
+// IsDeploymentWithManager return true if obj is a deployment containing a pod with at least one container named 'manager',
+// that according to the clusterctl contract, identifies the provider's controller.
+func IsDeploymentWithManager(obj unstructured.Unstructured) bool {
+	if obj.GroupVersionKind().Kind == deploymentKind {
+		var dep appsv1.Deployment
+		if err := runtime.DefaultUnstructuredConverter.FromUnstructured(obj.UnstructuredContent(), &dep); err != nil {
+			return false
+		}
+		for _, c := range dep.Spec.Template.Spec.Containers {
+			if c.Name == controllerContainerName {
+				return true
+			}
+		}
+	}
+	return false
+}

cmd/clusterctl/internal/util/objs_test.go

@@ -22,7 +22,11 @@ import (
 
 	. "github.com/onsi/gomega"
 
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 )
 
 func Test_inspectImages(t *testing.T) {
@@ -270,3 +274,92 @@ func TestFixImages(t *testing.T) {
 		})
 	}
 }
+
+func TestIsDeploymentWithManager(t *testing.T) {
+	convertor := runtime.DefaultUnstructuredConverter
+
+	depManagerContainer := &appsv1.Deployment{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Deployment",
+			APIVersion: "apps/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "manager-deployment",
+		},
+		Spec: appsv1.DeploymentSpec{
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{{Name: controllerContainerName}},
+				},
+			},
+		},
+	}
+	depManagerContainerObj, err := convertor.ToUnstructured(depManagerContainer)
+	if err != nil {
+		t.Fatalf("failed to construct unstructured object of %v: %v", depManagerContainer, err)
+	}
+
+	depNOManagerContainer := &appsv1.Deployment{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Deployment",
+			APIVersion: "apps/v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "not-manager-deployment",
+		},
+		Spec: appsv1.DeploymentSpec{
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{{Name: "not-manager"}},
+				},
+			},
+		},
+	}
+	depNOManagerContainerObj, err := convertor.ToUnstructured(depNOManagerContainer)
+	if err != nil {
+		t.Fatalf("failed to construct unstructured object of %v : %v", depNOManagerContainer, err)
+	}
+
+	svc := &corev1.Service{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Service",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-service",
+		},
+	}
+	svcObj, err := convertor.ToUnstructured(svc)
+	if err != nil {
+		t.Fatalf("failed to construct unstructured object of %v : %v", svc, err)
+	}
+
+	tests := []struct {
+		name     string
+		obj      unstructured.Unstructured
+		expected bool
+	}{
+		{
+			name:     "deployment with manager container",
+			obj:      unstructured.Unstructured{Object: depManagerContainerObj},
+			expected: true,
+		},
+		{
+			name:     "deployment without manager container",
+			obj:      unstructured.Unstructured{Object: depNOManagerContainerObj},
+			expected: false,
+		},
+		{
+			name:     "not a deployment",
+			obj:      unstructured.Unstructured{Object: svcObj},
+			expected: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			g := NewWithT(t)
+			actual := IsDeploymentWithManager(test.obj)
+			g.Expect(actual).To(Equal(test.expected))
+		})
+	}
+}