Skip to content

Commit

Permalink
✨ support proxy between hub cluster and managed cluster
Browse files Browse the repository at this point in the history
Signed-off-by: Yang Le <yangle@redhat.com>
  • Loading branch information
elgnay committed Sep 1, 2023
1 parent 36e389c commit 5cccebe
Show file tree
Hide file tree
Showing 11 changed files with 888 additions and 245 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ func (k *bootstrapController) sync(ctx context.Context, controllerContext factor
}

if bootstrapKubeconfig.Server != hubKubeconfig.Server ||
bootstrapKubeconfig.ProxyURL != hubKubeconfig.ProxyURL ||
!bytes.Equal(bootstrapKubeconfig.CertificateAuthorityData, hubKubeconfig.CertificateAuthorityData) {
// the bootstrap kubeconfig secret is changed, reload the klusterlet agents
reloadReason := fmt.Sprintf("the bootstrap secret %s/%s is changed", agentNamespace, helpers.BootstrapHubKubeConfig)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func TestSync(t *testing.T) {
name: "client certificate expired",
queueKey: "test/test",
objects: []runtime.Object{
newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443")),
newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443", "")),
newHubKubeConfigSecret("test", time.Now().Add(-60*time.Second).UTC()),
},
expectedRebootstrapping: true,
Expand All @@ -70,7 +70,7 @@ func TestSync(t *testing.T) {
{
name: "the bootstrap is not started",
queueKey: "test/test",
objects: []runtime.Object{newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443"))},
objects: []runtime.Object{newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443", ""))},
validateActions: func(t *testing.T, actions []clienttesting.Action) {
if len(actions) != 0 {
t.Errorf("expected no actions happens, but got %#v", actions)
Expand All @@ -81,7 +81,7 @@ func TestSync(t *testing.T) {
name: "the bootstrap secret is not changed",
queueKey: "test/test",
objects: []runtime.Object{
newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443")),
newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.47:6443", "")),
newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
},
validateActions: func(t *testing.T, actions []clienttesting.Action) {
Expand All @@ -91,10 +91,24 @@ func TestSync(t *testing.T) {
},
},
{
name: "the bootstrap secret is changed",
name: "hub server url is changed",
queueKey: "test/test",
objects: []runtime.Object{
newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443")),
newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443", "")),
newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
},
expectedRebootstrapping: true,
validateActions: func(t *testing.T, actions []clienttesting.Action) {
if len(actions) != 0 {
t.Errorf("expected no actions happens, but got %#v", actions)
}
},
},
{
name: "proxy url is changed",
queueKey: "test/test",
objects: []runtime.Object{
newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443", "https://10.0.118.10:3129")),
newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
},
expectedRebootstrapping: true,
Expand All @@ -109,7 +123,7 @@ func TestSync(t *testing.T) {
queueKey: "test/test",
initRebootstrapping: true,
objects: []runtime.Object{
newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443")),
newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443", "")),
newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
newDeploymentWithAvailableReplicas("test-registration-agent", "test", 1),
},
Expand All @@ -123,7 +137,7 @@ func TestSync(t *testing.T) {
queueKey: "test/test",
initRebootstrapping: true,
objects: []runtime.Object{
newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443")),
newSecret("bootstrap-hub-kubeconfig", "test", newKubeConfig("https://10.0.118.48:6443", "")),
newHubKubeConfigSecret("test", time.Now().Add(60*time.Second).UTC()),
newDeployment("test-registration-agent", "test"),
},
Expand Down Expand Up @@ -284,11 +298,12 @@ func newSecret(name, namespace string, kubeConfig []byte) *corev1.Secret {
return secret
}

func newKubeConfig(host string) []byte {
func newKubeConfig(host, proxyURL string) []byte {
configData, _ := runtime.Encode(clientcmdlatest.Codec, &clientcmdapi.Config{
Clusters: map[string]*clientcmdapi.Cluster{"default-cluster": {
Server: host,
InsecureSkipTLSVerify: true,
ProxyURL: proxyURL,
}},
Contexts: map[string]*clientcmdapi.Context{"default-context": {
Cluster: "default-cluster",
Expand Down Expand Up @@ -345,7 +360,7 @@ func newHubKubeConfigSecret(namespace string, notAfter time.Time) *corev1.Secret
Namespace: namespace,
},
Data: map[string][]byte{
"kubeconfig": newKubeConfig("https://10.0.118.47:6443"),
"kubeconfig": newKubeConfig("https://10.0.118.47:6443", ""),
"tls.crt": pem.EncodeToMemory(&pem.Block{
Type: certutil.CertificateBlockType,
Bytes: cert.Raw,
Expand Down
12 changes: 6 additions & 6 deletions pkg/registration/clientcert/certificate.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"k8s.io/client-go/kubernetes"
csrclient "k8s.io/client-go/kubernetes/typed/certificates/v1"
certificatesv1listers "k8s.io/client-go/listers/certificates/v1"
restclient "k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
certutil "k8s.io/client-go/util/cert"
Expand Down Expand Up @@ -146,19 +145,20 @@ func getCertValidityPeriod(secret *corev1.Secret) (*time.Time, *time.Time, error
}

// BuildKubeconfig builds a kubeconfig based on a rest config template with a cert/key pair
func BuildKubeconfig(clientConfig *restclient.Config, certPath, keyPath string) clientcmdapi.Config {
func BuildKubeconfig(server string, caData []byte, proxyURL, clientCertPath, clientKeyPath string) clientcmdapi.Config {
// Build kubeconfig.
kubeconfig := clientcmdapi.Config{
// Define a cluster stanza based on the bootstrap kubeconfig.
Clusters: map[string]*clientcmdapi.Cluster{"default-cluster": {
Server: clientConfig.Host,
Server: server,
InsecureSkipTLSVerify: false,
CertificateAuthorityData: clientConfig.CAData,
CertificateAuthorityData: caData,
ProxyURL: proxyURL,
}},
// Define auth based on the obtained client cert.
AuthInfos: map[string]*clientcmdapi.AuthInfo{"default-auth": {
ClientCertificate: certPath,
ClientKey: keyPath,
ClientCertificate: clientCertPath,
ClientKey: clientKeyPath,
}},
// Define a context that connects the auth info and cluster, and set it as the default
Contexts: map[string]*clientcmdapi.Context{"default-context": {
Expand Down
67 changes: 67 additions & 0 deletions pkg/registration/clientcert/certificate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package clientcert

import (
"crypto/x509/pkix"
"reflect"
"testing"
"time"

Expand Down Expand Up @@ -240,3 +241,69 @@ func TestGetCertValidityPeriod(t *testing.T) {
})
}
}

func TestBuildKubeconfig(t *testing.T) {
cases := []struct {
name string
server string
proxyURL string
caData []byte
clientCertFile string
clientKeyFile string
}{
{
name: "without proxy",
server: "https://127.0.0.1:6443",
caData: []byte("fake-ca-bundle"),
clientCertFile: "tls.crt",
clientKeyFile: "tls.key",
},
{
name: "with proxy",
server: "https://127.0.0.1:6443",
caData: []byte("fake-ca-bundle-with-proxy-ca"),
proxyURL: "https://127.0.0.1:3129",
clientCertFile: "tls.crt",
clientKeyFile: "tls.key",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
kubeconfig := BuildKubeconfig(c.server, c.caData, c.proxyURL, c.clientCertFile, c.clientKeyFile)
currentContext, ok := kubeconfig.Contexts[kubeconfig.CurrentContext]
if !ok {
t.Errorf("current context %q not found: %v", kubeconfig.CurrentContext, kubeconfig)
}

cluster, ok := kubeconfig.Clusters[currentContext.Cluster]
if !ok {
t.Errorf("cluster %q not found: %v", currentContext.Cluster, kubeconfig)
}

if cluster.Server != c.server {
t.Errorf("expected server %q, but got %q", c.server, cluster.Server)
}

if cluster.ProxyURL != c.proxyURL {
t.Errorf("expected proxy URL %q, but got %q", c.proxyURL, cluster.ProxyURL)
}

if !reflect.DeepEqual(cluster.CertificateAuthorityData, c.caData) {
t.Errorf("expected ca data %v, but got %v", c.caData, cluster.CertificateAuthorityData)
}

authInfo, ok := kubeconfig.AuthInfos[currentContext.AuthInfo]
if !ok {
t.Errorf("auth info %q not found: %v", currentContext.AuthInfo, kubeconfig)
}

if authInfo.ClientCertificate != c.clientCertFile {
t.Errorf("expected client certificate %q, but got %q", c.clientCertFile, authInfo.ClientCertificate)
}

if authInfo.ClientKey != c.clientKeyFile {
t.Errorf("expected client key %q, but got %q", c.clientKeyFile, authInfo.ClientKey)
}
})
}
}
33 changes: 31 additions & 2 deletions pkg/registration/spoke/spokeagent.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,12 @@ func (o *SpokeAgentConfig) RunSpokeAgentWithSpokeInformers(ctx context.Context,
managementKubeClient, 10*time.Minute, informers.WithNamespace(o.agentOptions.ComponentNamespace))

// create a kubeconfig with references to the key/cert files in the same secret
kubeconfig := clientcert.BuildKubeconfig(bootstrapClientConfig, clientcert.TLSCertFile, clientcert.TLSKeyFile)
proxyURL, err := getProxyURLFromKubeconfig(o.registrationOption.BootstrapKubeconfig)
if err != nil {
return err
}
kubeconfig := clientcert.BuildKubeconfig(bootstrapClientConfig.Host, bootstrapClientConfig.CAData, proxyURL,
clientcert.TLSCertFile, clientcert.TLSKeyFile)
kubeconfigData, err := clientcmd.Write(kubeconfig)
if err != nil {
return err
Expand Down Expand Up @@ -303,7 +308,12 @@ func (o *SpokeAgentConfig) RunSpokeAgentWithSpokeInformers(ctx context.Context,
recorder.Event("HubClientConfigReady", "Client config for hub is ready.")

// create a kubeconfig with references to the key/cert files in the same secret
kubeconfig := clientcert.BuildKubeconfig(hubClientConfig, clientcert.TLSCertFile, clientcert.TLSKeyFile)
proxyURL, err := getProxyURLFromKubeconfig(o.agentOptions.HubKubeconfigFile)
if err != nil {
return err
}
kubeconfig := clientcert.BuildKubeconfig(hubClientConfig.Host, hubClientConfig.CAData, proxyURL,
clientcert.TLSCertFile, clientcert.TLSKeyFile)
kubeconfigData, err := clientcmd.Write(kubeconfig)
if err != nil {
return err
Expand Down Expand Up @@ -465,3 +475,22 @@ func (o *SpokeAgentConfig) getSpokeClusterCABundle(kubeConfig *rest.Config) ([]b
}
return data, nil
}

func getProxyURLFromKubeconfig(filename string) (string, error) {
config, err := clientcmd.LoadFromFile(filename)
if err != nil {
return "", err
}

currentContext, ok := config.Contexts[config.CurrentContext]
if !ok {
return "", nil
}

cluster, ok := config.Clusters[currentContext.Cluster]
if !ok {
return "", nil
}

return cluster.ProxyURL, nil
}
48 changes: 48 additions & 0 deletions pkg/registration/spoke/spokeagent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,12 @@ import (
"time"

"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"

commonoptions "open-cluster-management.io/ocm/pkg/common/options"
testingcommon "open-cluster-management.io/ocm/pkg/common/testing"
"open-cluster-management.io/ocm/pkg/registration/clientcert"
testinghelpers "open-cluster-management.io/ocm/pkg/registration/helpers/testing"
)

Expand Down Expand Up @@ -231,3 +234,48 @@ func TestGetSpokeClusterCABundle(t *testing.T) {
})
}
}

func TestGetProxyURLFromKubeconfig(t *testing.T) {
tempDir, err := os.MkdirTemp("", "testgetproxyurl")
if err != nil {
t.Errorf("unexpected error: %v", err)
}
defer os.RemoveAll(tempDir)

kubeconfigWithoutProxy := clientcert.BuildKubeconfig("https://127.0.0.1:6443", nil, "", "tls.crt", "tls.key")
kubeconfigWithProxy := clientcert.BuildKubeconfig("https://127.0.0.1:6443", nil, "https://127.0.0.1:3129", "tls.crt", "tls.key")

cases := []struct {
name string
kubeconfig clientcmdapi.Config
expectedProxyURL string
}{
{
name: "without proxy url",
kubeconfig: kubeconfigWithoutProxy,
expectedProxyURL: "",
},
{
name: "with proxy url",
kubeconfig: kubeconfigWithProxy,
expectedProxyURL: "https://127.0.0.1:3129",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
filename := path.Join(tempDir, "kubeconfig")
if err := clientcmd.WriteToFile(c.kubeconfig, filename); err != nil {
t.Errorf("unexpected error: %v", err)
}

proxyURL, err := getProxyURLFromKubeconfig(filename)
if err != nil {
t.Errorf("unexpected error: %v", err)
}

if c.expectedProxyURL != proxyURL {
t.Errorf("expect %s, but %s", c.expectedProxyURL, proxyURL)
}
})
}
}
Loading

0 comments on commit 5cccebe

Please sign in to comment.