✨ support proxy between hub cluster and managed cluster

Signed-off-by: Yang Le <yangle@redhat.com>

pkg/operator/operators/klusterlet/controllers/bootstrapcontroller/bootstrapcontroller.go

@@ -154,6 +154,7 @@ func (k *bootstrapController) sync(ctx context.Context, controllerContext factor
 	}
 
 	if bootstrapKubeconfig.Server != hubKubeconfig.Server ||
+		bootstrapKubeconfig.ProxyURL != hubKubeconfig.ProxyURL ||
 		!bytes.Equal(bootstrapKubeconfig.CertificateAuthorityData, hubKubeconfig.CertificateAuthorityData) {
 		// the bootstrap kubeconfig secret is changed, reload the klusterlet agents
 		reloadReason := fmt.Sprintf("the bootstrap secret %s/%s is changed", agentNamespace, helpers.BootstrapHubKubeConfig)

pkg/operator/operators/klusterlet/controllers/bootstrapcontroller/bootstrapcontroller_test.go

@@ -57,7 +57,7 @@ func TestSync(t *testing.T) {
 			name:     "client certificate expired",
 			queueKey: "test/test",
 			objects: []runtime.Object{
-				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443")),
+				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443", "")),
 				newHubKubeConfigSecret("test", time.Now().Add(-60*time.Second).UTC()),
 			},
 			expectedRebootstrapping: true,
@@ -70,7 +70,7 @@ func TestSync(t *testing.T) {
 		{
 			name:     "the bootstrap is not started",
 			queueKey: "test/test",
-			objects:  []runtime.Object{newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443"))},
+			objects:  []runtime.Object{newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443", ""))},
 			validateActions: func(t *testing.T, actions []clienttesting.Action) {
 				if len(actions) != 0 {
 					t.Errorf("expected no actions happens, but got %#v", actions)
@@ -81,7 +81,7 @@ func TestSync(t *testing.T) {
 			name:     "the bootstrap secret is not changed",
 			queueKey: "test/test",
 			objects: []runtime.Object{
-				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443")),
+				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443", "")),
 				newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
 			},
 			validateActions: func(t *testing.T, actions []clienttesting.Action) {
@@ -91,10 +91,24 @@ func TestSync(t *testing.T) {
 			},
 		},
 		{
-			name:     "the bootstrap secret is changed",
+			name:     "hub server url is changed",
 			queueKey: "test/test",
 			objects: []runtime.Object{
-				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443")),
+				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443", "")),
+				newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
+			},
+			expectedRebootstrapping: true,
+			validateActions: func(t *testing.T, actions []clienttesting.Action) {
+				if len(actions) != 0 {
+					t.Errorf("expected no actions happens, but got %#v", actions)
+				}
+			},
+		},
+		{
+			name:     "proxy url is changed",
+			queueKey: "test/test",
+			objects: []runtime.Object{
+				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443", "https://10.0.118.10:3129")),
 				newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
 			},
 			expectedRebootstrapping: true,
@@ -109,7 +123,7 @@ func TestSync(t *testing.T) {
 			queueKey:            "test/test",
 			initRebootstrapping: true,
 			objects: []runtime.Object{
-				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443")),
+				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443", "")),
 				newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
 				newDeploymentWithAvailableReplicas("test-registration-agent", "test", 1),
 			},
@@ -123,7 +137,7 @@ func TestSync(t *testing.T) {
 			queueKey:            "test/test",
 			initRebootstrapping: true,
 			objects: []runtime.Object{
-				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443")),
+				newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443", "")),
 				newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
 				newDeployment("test-registration-agent", "test"),
 			},
@@ -284,11 +298,12 @@ func newSecret(name, namespace string, kubeConfig []byte) *corev1.Secret {
 	return secret
 }
 
-func newKubeConfig(host string) []byte {
+func newKubeConfig(host, proxyURL string) []byte {
 	configData, _ := runtime.Encode(clientcmdlatest.Codec, &clientcmdapi.Config{
 		Clusters: map[string]*clientcmdapi.Cluster{"default-cluster": {
 			Server:                host,
 			InsecureSkipTLSVerify: true,
+			ProxyURL:              proxyURL,
 		}},
 		Contexts: map[string]*clientcmdapi.Context{"default-context": {
 			Cluster: "default-cluster",
@@ -345,7 +360,7 @@ func newHubKubeConfigSecret(namespace string, notAfter time.Time) *corev1.Secret
 			Namespace: namespace,
 		},
 		Data: map[string][]byte{
-			"kubeconfig": newKubeConfig("https://10.0.118.47:6443"),
+			"kubeconfig": newKubeConfig("https://10.0.118.47:6443", ""),
 			"tls.crt": pem.EncodeToMemory(&pem.Block{
 				Type:  certutil.CertificateBlockType,
 				Bytes: cert.Raw,

pkg/registration/clientcert/certificate.go

@@ -17,7 +17,6 @@ import (
 	"k8s.io/client-go/kubernetes"
 	csrclient "k8s.io/client-go/kubernetes/typed/certificates/v1"
 	certificatesv1listers "k8s.io/client-go/listers/certificates/v1"
-	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/cache"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 	certutil "k8s.io/client-go/util/cert"
@@ -146,19 +145,20 @@ func getCertValidityPeriod(secret *corev1.Secret) (*time.Time, *time.Time, error
 }
 
 // BuildKubeconfig builds a kubeconfig based on a rest config template with a cert/key pair
-func BuildKubeconfig(clientConfig *restclient.Config, certPath, keyPath string) clientcmdapi.Config {
+func BuildKubeconfig(server string, caData []byte, proxyURL, clientCertPath, clientKeyPath string) clientcmdapi.Config {
 	// Build kubeconfig.
 	kubeconfig := clientcmdapi.Config{
 		// Define a cluster stanza based on the bootstrap kubeconfig.
 		Clusters: map[string]*clientcmdapi.Cluster{"default-cluster": {
-			Server:                   clientConfig.Host,
+			Server:                   server,
 			InsecureSkipTLSVerify:    false,
-			CertificateAuthorityData: clientConfig.CAData,
+			CertificateAuthorityData: caData,
+			ProxyURL:                 proxyURL,
 		}},
 		// Define auth based on the obtained client cert.
 		AuthInfos: map[string]*clientcmdapi.AuthInfo{"default-auth": {
-			ClientCertificate: certPath,
-			ClientKey:         keyPath,
+			ClientCertificate: clientCertPath,
+			ClientKey:         clientKeyPath,
 		}},
 		// Define a context that connects the auth info and cluster, and set it as the default
 		Contexts: map[string]*clientcmdapi.Context{"default-context": {

pkg/registration/clientcert/certificate_test.go

@@ -2,6 +2,7 @@ package clientcert
 
 import (
 	"crypto/x509/pkix"
+	"reflect"
 	"testing"
 	"time"
 
@@ -240,3 +241,69 @@ func TestGetCertValidityPeriod(t *testing.T) {
 		})
 	}
 }
+
+func TestBuildKubeconfig(t *testing.T) {
+	cases := []struct {
+		name           string
+		server         string
+		proxyURL       string
+		caData         []byte
+		clientCertFile string
+		clientKeyFile  string
+	}{
+		{
+			name:           "without proxy",
+			server:         "https://127.0.0.1:6443",
+			caData:         []byte("fake-ca-bundle"),
+			clientCertFile: "tls.crt",
+			clientKeyFile:  "tls.key",
+		},
+		{
+			name:           "with proxy",
+			server:         "https://127.0.0.1:6443",
+			caData:         []byte("fake-ca-bundle-with-proxy-ca"),
+			proxyURL:       "https://127.0.0.1:3129",
+			clientCertFile: "tls.crt",
+			clientKeyFile:  "tls.key",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			kubeconfig := BuildKubeconfig(c.server, c.caData, c.proxyURL, c.clientCertFile, c.clientKeyFile)
+			currentContext, ok := kubeconfig.Contexts[kubeconfig.CurrentContext]
+			if !ok {
+				t.Errorf("current context %q not found: %v", kubeconfig.CurrentContext, kubeconfig)
+			}
+
+			cluster, ok := kubeconfig.Clusters[currentContext.Cluster]
+			if !ok {
+				t.Errorf("cluster %q not found: %v", currentContext.Cluster, kubeconfig)
+			}
+
+			if cluster.Server != c.server {
+				t.Errorf("expected server %q, but got %q", c.server, cluster.Server)
+			}
+
+			if cluster.ProxyURL != c.proxyURL {
+				t.Errorf("expected proxy URL %q, but got %q", c.proxyURL, cluster.ProxyURL)
+			}
+
+			if !reflect.DeepEqual(cluster.CertificateAuthorityData, c.caData) {
+				t.Errorf("expected ca data %v, but got %v", c.caData, cluster.CertificateAuthorityData)
+			}
+
+			authInfo, ok := kubeconfig.AuthInfos[currentContext.AuthInfo]
+			if !ok {
+				t.Errorf("auth info %q not found: %v", currentContext.AuthInfo, kubeconfig)
+			}
+
+			if authInfo.ClientCertificate != c.clientCertFile {
+				t.Errorf("expected client certificate %q, but got %q", c.clientCertFile, authInfo.ClientCertificate)
+			}
+
+			if authInfo.ClientKey != c.clientKeyFile {
+				t.Errorf("expected client key %q, but got %q", c.clientKeyFile, authInfo.ClientKey)
+			}
+		})
+	}
+}

pkg/registration/spoke/spokeagent.go

@@ -213,7 +213,12 @@ func (o *SpokeAgentConfig) RunSpokeAgentWithSpokeInformers(ctx context.Context,
 			managementKubeClient, 10*time.Minute, informers.WithNamespace(o.agentOptions.ComponentNamespace))
 
 		// create a kubeconfig with references to the key/cert files in the same secret
-		kubeconfig := clientcert.BuildKubeconfig(bootstrapClientConfig, clientcert.TLSCertFile, clientcert.TLSKeyFile)
+		proxyURL, err := getProxyURLFromKubeconfig(o.registrationOption.BootstrapKubeconfig)
+		if err != nil {
+			return err
+		}
+		kubeconfig := clientcert.BuildKubeconfig(bootstrapClientConfig.Host, bootstrapClientConfig.CAData, proxyURL,
+			clientcert.TLSCertFile, clientcert.TLSKeyFile)
 		kubeconfigData, err := clientcmd.Write(kubeconfig)
 		if err != nil {
 			return err
@@ -303,7 +308,12 @@ func (o *SpokeAgentConfig) RunSpokeAgentWithSpokeInformers(ctx context.Context,
 	recorder.Event("HubClientConfigReady", "Client config for hub is ready.")
 
 	// create a kubeconfig with references to the key/cert files in the same secret
-	kubeconfig := clientcert.BuildKubeconfig(hubClientConfig, clientcert.TLSCertFile, clientcert.TLSKeyFile)
+	proxyURL, err := getProxyURLFromKubeconfig(o.agentOptions.HubKubeconfigFile)
+	if err != nil {
+		return err
+	}
+	kubeconfig := clientcert.BuildKubeconfig(hubClientConfig.Host, hubClientConfig.CAData, proxyURL,
+		clientcert.TLSCertFile, clientcert.TLSKeyFile)
 	kubeconfigData, err := clientcmd.Write(kubeconfig)
 	if err != nil {
 		return err
@@ -465,3 +475,22 @@ func (o *SpokeAgentConfig) getSpokeClusterCABundle(kubeConfig *rest.Config) ([]b
 	}
 	return data, nil
 }
+
+func getProxyURLFromKubeconfig(filename string) (string, error) {
+	config, err := clientcmd.LoadFromFile(filename)
+	if err != nil {
+		return "", err
+	}
+
+	currentContext, ok := config.Contexts[config.CurrentContext]
+	if !ok {
+		return "", nil
+	}
+
+	cluster, ok := config.Clusters[currentContext.Cluster]
+	if !ok {
+		return "", nil
+	}
+
+	return cluster.ProxyURL, nil
+}

pkg/registration/spoke/spokeagent_test.go

@@ -9,9 +9,12 @@ import (
 	"time"
 
 	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 
 	commonoptions "open-cluster-management.io/ocm/pkg/common/options"
 	testingcommon "open-cluster-management.io/ocm/pkg/common/testing"
+	"open-cluster-management.io/ocm/pkg/registration/clientcert"
 	testinghelpers "open-cluster-management.io/ocm/pkg/registration/helpers/testing"
 )
 
@@ -231,3 +234,48 @@ func TestGetSpokeClusterCABundle(t *testing.T) {
 		})
 	}
 }
+
+func TestGetProxyURLFromKubeconfig(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "testgetproxyurl")
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	kubeconfigWithoutProxy := clientcert.BuildKubeconfig("https://127.0.0.1:6443", nil, "", "tls.crt", "tls.key")
+	kubeconfigWithProxy := clientcert.BuildKubeconfig("https://127.0.0.1:6443", nil, "https://127.0.0.1:3129", "tls.crt", "tls.key")
+
+	cases := []struct {
+		name             string
+		kubeconfig       clientcmdapi.Config
+		expectedProxyURL string
+	}{
+		{
+			name:             "without proxy url",
+			kubeconfig:       kubeconfigWithoutProxy,
+			expectedProxyURL: "",
+		},
+		{
+			name:             "with proxy url",
+			kubeconfig:       kubeconfigWithProxy,
+			expectedProxyURL: "https://127.0.0.1:3129",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			filename := path.Join(tempDir, "kubeconfig")
+			if err := clientcmd.WriteToFile(c.kubeconfig, filename); err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+
+			proxyURL, err := getProxyURLFromKubeconfig(filename)
+			if err != nil {
+				t.Errorf("unexpected error: %v", err)
+			}
+
+			if c.expectedProxyURL != proxyURL {
+				t.Errorf("expect %s, but %s", c.expectedProxyURL, proxyURL)
+			}
+		})
+	}
+}