Support multiple bootstrapkubeconfigs in registration-agent.

Signed-off-by: xuezhaojun <zxue@redhat.com>

pkg/registration/spoke/bootstrapkubeconfigsmanager/bootstrapkubeconfig.go

@@ -0,0 +1,106 @@
+package bootstrapkubeconfigsmanager
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+type boostrapKubeConfigStatus string
+
+const (
+	boostrapKubeConfigStatusInValid boostrapKubeConfigStatus = "InValid"
+	boostrapKubeConfigStatusValid   boostrapKubeConfigStatus = "Valid"
+)
+
+// bootstrapKubeConfig represents a bootstrap kubeconfig that agent can use to bootstrap a managed cluster.
+type boostrapKubeConfig interface {
+	// Name returns the name of the bootstrap kubeconfig. It helps to identify the bootstrap kubeconfig.
+	Name() string
+
+	// KubeConfigData returns the kubeconfig data of the bootstrap kubeconfig.
+	KubeConfigData() ([]byte, error)
+
+	// Status returns the status of the bootstrap kubeconfig.
+	// A bootstrap kubeconfig has two status: Valid and InValid.
+	Status() (boostrapKubeConfigStatus, error)
+
+	// Fail means at the time t, the bootstrap kubeconfig failed to connect to the hub cluster.
+	Fail(t time.Time) error
+}
+
+var _ boostrapKubeConfig = &boostrapKubeConfigSecretImpl{}
+
+const (
+	// BootstrapKubeconfigFailedTimeAnnotationKey represents the time when the bootstrap kubeconfig failed
+	BootstrapKubeconfigFailedTimeAnnotationKey = "agent.open-cluster-management.io/bootstrap-kubeconfig-failed-time"
+)
+
+type boostrapKubeConfigSecretImpl struct {
+	secretName                           string
+	secretNamespace                      string
+	skipFailedBootstrapKubeconfigSeconds int32 // if a bootstrap kubeconfig failed, in 3 mins, it can't be used in rebootstrap.
+	kubeClient                           kubernetes.Interface
+}
+
+func (b *boostrapKubeConfigSecretImpl) Name() string {
+	return b.secretName
+}
+
+func (b *boostrapKubeConfigSecretImpl) KubeConfigData() ([]byte, error) {
+	secret, err := b.kubeClient.CoreV1().Secrets(b.secretNamespace).Get(context.Background(), b.secretName, metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("get the bootstrap kubeconfig secret failed: %v", err)
+	}
+	if secret.Data == nil {
+		return nil, fmt.Errorf("the bootstrap kubeconfig secret %s has no data", b.secretName)
+	}
+	kubeconfigData, ok := secret.Data["kubeconfig"]
+	if !ok {
+		return nil, fmt.Errorf("the bootstrap kubeconfig secret %s has no kubeconfig data", b.secretName)
+	}
+	return kubeconfigData, nil
+}
+
+func (b *boostrapKubeConfigSecretImpl) Status() (boostrapKubeConfigStatus, error) {
+	secret, err := b.kubeClient.CoreV1().Secrets(b.secretNamespace).Get(context.Background(), b.secretName, metav1.GetOptions{})
+	if err != nil {
+		return boostrapKubeConfigStatusInValid, fmt.Errorf("get the bootstrap kubeconfig secret failed: %v", err)
+	}
+
+	if secret.Annotations == nil {
+		return boostrapKubeConfigStatusValid, nil
+	}
+
+	now := time.Now()
+	if failedTime, ok := secret.Annotations[BootstrapKubeconfigFailedTimeAnnotationKey]; ok {
+		failedTimeParsed, err := time.Parse(time.RFC3339, failedTime)
+		if err != nil {
+			return boostrapKubeConfigStatusInValid, fmt.Errorf("failed to parse the failed time %s of the secret %s: %v", failedTime, secret.Name, err)
+		}
+		if now.Sub(failedTimeParsed).Seconds() < float64(b.skipFailedBootstrapKubeconfigSeconds) {
+			return boostrapKubeConfigStatusInValid, nil
+		}
+	}
+	return boostrapKubeConfigStatusValid, nil
+}
+
+func (b *boostrapKubeConfigSecretImpl) Fail(t time.Time) error {
+	secret, err := b.kubeClient.CoreV1().Secrets(b.secretNamespace).Get(context.Background(), b.secretName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("get the bootstrap kubeconfig secret failed: %v", err)
+	}
+	secretCopy := secret.DeepCopy()
+	if secretCopy.Annotations == nil {
+		secretCopy.Annotations = make(map[string]string)
+	}
+	secretCopy.Annotations[BootstrapKubeconfigFailedTimeAnnotationKey] = t.Format(time.RFC3339)
+	_, err = b.kubeClient.CoreV1().Secrets(b.secretNamespace).Update(context.Background(), secretCopy, metav1.UpdateOptions{})
+	if err != nil {
+		return fmt.Errorf("update the secret %s failed: %v", b.secretName, err)
+	}
+	return nil
+}

pkg/registration/spoke/bootstrapkubeconfigsmanager/bootstrapkubeconfig_test.go

@@ -0,0 +1,202 @@
+package bootstrapkubeconfigsmanager
+
+import (
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/fake"
+	k8stesting "k8s.io/client-go/testing"
+)
+
+func TestBootstrapKubeConfigSecretImpl(t *testing.T) {
+	testcases := []struct {
+		name       string
+		mockClient func(failedTime time.Time) *fake.Clientset
+		validate   func(c boostrapKubeConfig)
+	}{
+		{
+			name: "Valid BootstrapKubeConfig",
+			mockClient: func(_ time.Time) *fake.Clientset {
+				// Create a mock secret
+				mockSecret := &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "mock-secret",
+						Namespace: "test-namespace",
+					},
+					Data: map[string][]byte{
+						"kubeconfig": []byte("mock-kubeconfig"),
+					},
+				}
+
+				// Create a mock kubeClient
+				mockKubeClient := &fake.Clientset{}
+				mockKubeClient.AddReactor("get", "secrets", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
+					return true, mockSecret, nil
+				})
+				return mockKubeClient
+			},
+			validate: func(c boostrapKubeConfig) {
+				if c.Name() != "mock-secret" {
+					t.Errorf("Expected name %v, but got %v", "mock-secret", c.Name())
+				}
+
+				kubeConfigData, err := c.KubeConfigData()
+				if err != nil {
+					t.Errorf("Unexpected error: %v", err)
+				}
+
+				if string(kubeConfigData) != "mock-kubeconfig" {
+					t.Errorf("Expected kubeconfig data %v, but got %v", "mock-kubeconfig", string(kubeConfigData))
+				}
+
+				status, err := c.Status()
+				if err != nil {
+					t.Errorf("Unexpected error: %v", err)
+				}
+
+				if status != boostrapKubeConfigStatusValid {
+					t.Errorf("Expected status %v, but got %v", boostrapKubeConfigStatusValid, status)
+				}
+			},
+		},
+		{
+			name: "Once failed but now valid BootstrapKubeConfig",
+			mockClient: func(_ time.Time) *fake.Clientset {
+				// Create a mock secret
+				mockSecret := &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "mock-secret",
+						Namespace: "test-namespace",
+						Annotations: map[string]string{
+							BootstrapKubeconfigFailedTimeAnnotationKey: "2021-08-01T00:00:00Z",
+						},
+					},
+					Data: map[string][]byte{
+						"kubeconfig": []byte("mock-kubeconfig"),
+					},
+				}
+
+				// Create a mock kubeClient
+				mockKubeClient := &fake.Clientset{}
+				mockKubeClient.AddReactor("get", "secrets", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
+					return true, mockSecret, nil
+				})
+				return mockKubeClient
+			},
+			validate: func(c boostrapKubeConfig) {
+				status, err := c.Status()
+				if err != nil {
+					t.Errorf("Unexpected error: %v", err)
+				}
+
+				if status != boostrapKubeConfigStatusValid {
+					t.Errorf("Expected status %v, but got %v", boostrapKubeConfigStatusValid, status)
+				}
+			},
+		},
+		{
+			name: "Recently failed and invalid BootstrapKubeConfig",
+			mockClient: func(failedTime time.Time) *fake.Clientset {
+				// Create a mock secret
+				mockSecret := &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "mock-secret",
+						Namespace: "test-namespace",
+						Annotations: map[string]string{
+							BootstrapKubeconfigFailedTimeAnnotationKey: failedTime.Format(time.RFC3339),
+						},
+					},
+					Data: map[string][]byte{
+						"kubeconfig": []byte("mock-kubeconfig"),
+					},
+				}
+
+				// Create a mock kubeClient
+				mockKubeClient := &fake.Clientset{}
+				mockKubeClient.AddReactor("get", "secrets", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
+					return true, mockSecret, nil
+				})
+				return mockKubeClient
+			},
+			validate: func(c boostrapKubeConfig) {
+				status, err := c.Status()
+				if err != nil {
+					t.Errorf("Unexpected error: %v", err)
+				}
+
+				if status != boostrapKubeConfigStatusInValid {
+					t.Errorf("Expected status %v, but got %v", boostrapKubeConfigStatusInValid, status)
+				}
+			},
+		},
+		{
+			name: "Fail a BootstrapKubeConfig",
+			mockClient: func(_ time.Time) *fake.Clientset {
+				// Create a mock secret
+				mockSecret := &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "mock-secret",
+						Namespace: "test-namespace",
+					},
+					Data: map[string][]byte{
+						"kubeconfig": []byte("mock-kubeconfig"),
+					},
+				}
+
+				// Create a mock kubeClient
+				mockKubeClient := &fake.Clientset{}
+				mockKubeClient.AddReactor("get", "secrets", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
+					return true, mockSecret, nil
+				})
+				mockKubeClient.AddReactor("update", "secrets", func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) {
+					// Get annotation from action
+					annotations := action.(k8stesting.UpdateAction).GetObject().(*corev1.Secret).Annotations
+					mockSecret.Annotations = annotations
+					return true, nil, nil
+				})
+				return mockKubeClient
+			},
+			validate: func(c boostrapKubeConfig) {
+				status, err := c.Status()
+				if err != nil {
+					t.Errorf("Unexpected error: %v", err)
+				}
+
+				if status != boostrapKubeConfigStatusValid {
+					t.Errorf("Expected status %v, but got %v", boostrapKubeConfigStatusValid, status)
+				}
+
+				// Fail the BootstrapKubeConfig
+				err = c.Fail(time.Now())
+				if err != nil {
+					t.Errorf("Unexpected error: %v", err)
+				}
+
+				status, err = c.Status()
+				if err != nil {
+					t.Errorf("Unexpected error: %v", err)
+				}
+
+				if status != boostrapKubeConfigStatusInValid {
+					t.Errorf("Expected status %v, but got %v", boostrapKubeConfigStatusInValid, status)
+				}
+			},
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			mockClient := tc.mockClient(time.Now())
+			bootstrapKubeConfig := &boostrapKubeConfigSecretImpl{
+				kubeClient:                           mockClient,
+				secretNamespace:                      "test-namespace",
+				secretName:                           "mock-secret",
+				skipFailedBootstrapKubeconfigSeconds: 60,
+			}
+			tc.validate(bootstrapKubeConfig)
+		})
+	}
+}

pkg/registration/spoke/bootstrapkubeconfigsmanager/bootstrapkubeconfiginuse.go

@@ -0,0 +1,60 @@
+package bootstrapkubeconfigsmanager
+
+import (
+	"context"
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+// bootstrapKubeConfigInUse is the registration spoke's current in used bootstrap kubeconfig.
+type bootstrapKubeConfigInUse interface {
+	// KubeConfigData returns the kubeconfig data of the bootstrap kubeconfig in use.
+	KubeConfigData() ([]byte, error)
+
+	// Update updates the kubeconfig data of the bootstrap kubeconfig in use.
+	Update(ctx context.Context, kubeconfigData []byte) error
+}
+
+var _ bootstrapKubeConfigInUse = &bootstrapKubeConfigInUseImpl{}
+
+type bootstrapKubeConfigInUseImpl struct {
+	secretName      string
+	secretNamespace string
+	kubeClient      kubernetes.Interface
+}
+
+func (b *bootstrapKubeConfigInUseImpl) KubeConfigData() ([]byte, error) {
+	secret, err := b.kubeClient.CoreV1().Secrets(b.secretNamespace).Get(context.Background(), b.secretName, metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("get the bootstrap kubeconfig secret failed: %v", err)
+	}
+	if secret.Data == nil {
+		return nil, fmt.Errorf("the bootstrap kubeconfig secret %s has no data", b.secretName)
+	}
+	kubeconfigData, ok := secret.Data["kubeconfig"]
+	if !ok {
+		return nil, fmt.Errorf("the bootstrap kubeconfig secret %s has no kubeconfig data", b.secretName)
+	}
+	return kubeconfigData, nil
+}
+
+func (b *bootstrapKubeConfigInUseImpl) Update(ctx context.Context, kubeconfigData []byte) error {
+	var err error
+	// Get the in-use bootstrapkubeconfig secret
+	inUse, err := b.kubeClient.CoreV1().Secrets(b.secretNamespace).Get(ctx, b.secretName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("get the current bootstrap kubeconfig secret failed: %v", err)
+	}
+
+	// Update the in-use bootstrapkubeconfig secret
+	copy := inUse.DeepCopy()
+	copy.Data["kubeconfig"] = kubeconfigData
+
+	_, err = b.kubeClient.CoreV1().Secrets(b.secretNamespace).Update(ctx, copy, metav1.UpdateOptions{})
+	if err != nil {
+		return fmt.Errorf("update the current bootstrap kubeconfig secret failed: %v", err)
+	}
+	return nil
+}