Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(operator): fix tls rotation logic for controller #1757

Merged
merged 1 commit into from
May 29, 2024
+47 −102
Merged

fix(operator): fix tls rotation logic for controller #1757

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pkg/KubeArmorOperator/internal/controller/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -701,7 +701,7 @@ func (clusterWatcher *ClusterWatcher) DeleteAllTlsSecrets() error {
for _, secret := range tlsSecrets {
err := clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Delete(context.Background(), secret, v1.DeleteOptions{})
if err != nil {
clusterWatcher.Log.Errorf("error while deleing secret: %s", secret)
clusterWatcher.Log.Errorf("error while deleing secret: %s, error=%s", secret, err.Error())
return err
}
}
Expand Down
147 changes: 46 additions & 101 deletions pkg/KubeArmorOperator/internal/controller/resources.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ func generateDaemonset(name, enforcer, runtime, socket, btfPresent, apparmorfs,
vols = append(vols, common.KubeArmorCaVolume...)
volMnts = append(volMnts, common.KubeArmorCaVolumeMount...)
common.AddOrReplaceArg("-tlsEnabled=true", "-tlsEnabled=false", &daemonset.Spec.Template.Spec.Containers[0].Args)
} else {
common.AddOrReplaceArg("-tlsEnabled=false", "-tlsEnabled=true", &daemonset.Spec.Template.Spec.Containers[0].Args)
}
daemonset.Spec.Template.Spec.Volumes = vols
daemonset.Spec.Template.Spec.InitContainers[0].VolumeMounts = commonVolMnts
Expand Down Expand Up @@ -426,7 +428,6 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() {
var caCert, tlsCrt, tlsKey *bytes.Buffer
var kGenErr, err, installErr error
RotateTls := false
FirstRun := true
srvAccs := []*corev1.ServiceAccount{
addOwnership(deployments.GetServiceAccount(common.Namespace)).(*corev1.ServiceAccount),
addOwnership(deployments.GetRelayServiceAccount(common.Namespace)).(*corev1.ServiceAccount),
Expand Down Expand Up @@ -486,6 +487,8 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() {
relayServer.Spec.Template.Spec.Volumes =
append(relayServer.Spec.Template.Spec.Volumes, common.KubeArmorRelayTlsVolume...)
common.AddOrReplaceArg("-tlsEnabled=true", "-tlsEnabled=false", &relayServer.Spec.Template.Spec.Containers[0].Args)
} else {
common.AddOrReplaceArg("-tlsEnabled=false", "-tlsEnabled=true", &relayServer.Spec.Template.Spec.Containers[0].Args)
}

// update images
Expand Down Expand Up @@ -631,10 +634,7 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() {
installErr = err
clusterWatcher.Log.Warnf("Cannot create secret %s, error=%s", secret.Name, err.Error())
} else {
RotateTls = true && !FirstRun
if !FirstRun {
clusterWatcher.Log.Warnf("Secret was created, rotating TLS secrets")
}
caInK8sSecret = secret.Data["ca.crt"]
}
} else if err == nil {
caInK8sSecret = s.Data["ca.crt"]
Expand Down Expand Up @@ -672,10 +672,7 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() {
} else if err == nil {
if !bytes.Equal(hook.Webhooks[0].ClientConfig.CABundle, caInK8sSecret) {
// rotate
RotateTls = true && !FirstRun
if !FirstRun {
clusterWatcher.Log.Warnf("mutation CA cert does not match secret CA cert, rotating tls secrets")
}
RotateTls = true
}
} else {
installErr = err
Expand All @@ -698,107 +695,55 @@ func (clusterWatcher *ClusterWatcher) WatchRequiredResources() {
clusterWatcher.RotateTlsCerts()
RotateTls = false
}
if FirstRun {
FirstRun = false
}
time.Sleep(10 * time.Second)
}
}

func (clusterWatcher *ClusterWatcher) RotateTlsCerts() {
var suffix string
var caCert, tlsCrt, tlsKey *bytes.Buffer
var err error
retries := 0
for {
if retries == 3 {
return
}
retries++
suffix, err = common.GetFreeRandSuffix(clusterWatcher.Client, common.Namespace)
if err == nil {
clusterWatcher.Log.Infof("Using suffix %s for all new temorary resources", suffix)
break
}
clusterWatcher.Log.Infof("Cannot find a suffix, err=%s, retrying in 3 seconds...", err.Error())
time.Sleep(3 * time.Second)
}
serviceName := deployments.KubeArmorControllerWebhookServiceName + "-" + suffix
for {
caCert, tlsCrt, tlsKey, err = common.GeneratePki(common.Namespace, serviceName)
if err == nil {
break
}
clusterWatcher.Log.Infof("Couldn't generate TLS secret, retrying in 3 seconds")
time.Sleep(3 * time.Second)
}
tmpsecret := deployments.GetKubeArmorControllerTLSSecret(common.Namespace, caCert.String(), tlsCrt.String(), tlsKey.String())
tmpsecret = addOwnership(tmpsecret).(*corev1.Secret)
tmpsecret.Name = tmpsecret.GetName() + "-" + suffix
_, err = clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Create(context.Background(), tmpsecret, metav1.CreateOptions{})

origdeploy, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Get(context.Background(), deployments.KubeArmorControllerDeploymentName, metav1.GetOptions{})
if err != nil {
clusterWatcher.Log.Warnf("Cannot create secret %s, error=%s", tmpsecret.Name, err.Error())
}
tmpdeploy := deployments.GetKubeArmorControllerDeployment(common.Namespace)
tmpdeploy = addOwnership(tmpdeploy).(*appsv1.Deployment)
tmpdeploy.Name = tmpdeploy.GetName() + "-" + suffix
for i, s := range tmpdeploy.Spec.Template.Spec.Volumes {
if s.Name == "cert" {
s.Secret.SecretName = tmpsecret.GetName()
tmpdeploy.Spec.Template.Spec.Volumes[i] = s
break
}
clusterWatcher.Log.Warnf("cannot get controller deployment, error=%s", err.Error())
}
selectLabels := tmpdeploy.Spec.Selector.MatchLabels
selectLabels["kubearmor-app"] = suffix
tmpdeploy.Spec.Selector.MatchLabels = selectLabels
origdeploy, _ := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Get(context.Background(), deployments.KubeArmorControllerDeploymentName, metav1.GetOptions{})
origdeploy = addOwnership(origdeploy).(*appsv1.Deployment)
tmpdeploy.Spec.Replicas = origdeploy.Spec.Replicas
if _, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Create(context.Background(), tmpdeploy, metav1.CreateOptions{}); err != nil {
clusterWatcher.Log.Warnf("Cannot create deployment %s, error=%s", tmpdeploy.Name, err.Error())
}

time.Sleep(10 * time.Second)

tmpservice := deployments.GetKubeArmorControllerWebhookService(common.Namespace)
tmpservice = addOwnership(tmpservice).(*corev1.Service)
tmpservice.Name = serviceName
tmpservice.Spec.Selector = selectLabels
if _, err := clusterWatcher.Client.CoreV1().Services(common.Namespace).Create(context.Background(), tmpservice, metav1.CreateOptions{}); err != nil {
clusterWatcher.Log.Warnf("Cannot create deployment %s, error=%s", tmpservice.Name, err.Error())
}
tmpmutation := deployments.GetKubeArmorControllerMutationAdmissionConfiguration(common.Namespace, caCert.Bytes())
mutationName := tmpmutation.Name
tmpmutation = addOwnership(tmpmutation).(*v1.MutatingWebhookConfiguration)
tmpmutation.Name = tmpmutation.Name + "-" + suffix
tmpmutation.Webhooks[0].ClientConfig.Service.Name = tmpservice.GetName()
if _, err := clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.Background(), tmpmutation, metav1.CreateOptions{}); err != nil {
clusterWatcher.Log.Warnf("Cannot create mutation webhook %s, error=%s", tmpmutation.Name, err.Error())
}
clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), mutationName, metav1.DeleteOptions{})

caCert, tlsCrt, tlsKey, _ = common.GeneratePki(common.Namespace, deployments.KubeArmorControllerWebhookServiceName)
secret := deployments.GetKubeArmorControllerTLSSecret(common.Namespace, caCert.String(), tlsCrt.String(), tlsKey.String())
secret = addOwnership(secret).(*corev1.Secret)
clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Update(context.Background(), secret, metav1.UpdateOptions{})

replicas := int32(0)
origdeploy.Spec.Replicas = &replicas
clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Update(context.Background(), origdeploy, metav1.UpdateOptions{})
time.Sleep(10 * time.Second)
origdeploy, _ = clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Get(context.Background(), deployments.KubeArmorControllerDeploymentName, metav1.GetOptions{})
origdeploy = addOwnership(origdeploy).(*appsv1.Deployment)
origdeploy.Spec.Replicas = tmpdeploy.Spec.Replicas

clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Update(context.Background(), origdeploy, metav1.UpdateOptions{})
mutation := deployments.GetKubeArmorControllerMutationAdmissionConfiguration(common.Namespace, caCert.Bytes())
mutation = addOwnership(mutation).(*v1.MutatingWebhookConfiguration)

clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.Background(), mutation, metav1.CreateOptions{})

clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), tmpmutation.Name, metav1.DeleteOptions{})
clusterWatcher.Client.CoreV1().Services(common.Namespace).Delete(context.Background(), tmpservice.Name, metav1.DeleteOptions{})
clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Delete(context.Background(), tmpdeploy.Name, metav1.DeleteOptions{})
clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Delete(context.Background(), tmpsecret.Name, metav1.DeleteOptions{})
replicas := origdeploy.Spec.Replicas

// TODO: Keep CA certificate in k8s secret

// == CLEANUP ==
// scale down controller deployment to 0
controllerDeployment := deployments.GetKubeArmorControllerDeployment(common.Namespace)
controllerDeployment = addOwnership(controllerDeployment).(*appsv1.Deployment)
zeroReplicas := int32(0)
controllerDeployment.Spec.Replicas = &zeroReplicas
if _, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Update(context.Background(), controllerDeployment, metav1.UpdateOptions{}); err != nil {
clusterWatcher.Log.Warnf("cannot scale down controller %s, error=%s", controllerDeployment.Name, err.Error())
}
// delete mutation webhook configuration
mutationWebhook := deployments.GetKubeArmorControllerMutationAdmissionConfiguration(common.Namespace, caCert.Bytes())
mutationWebhook = addOwnership(mutationWebhook).(*v1.MutatingWebhookConfiguration)
if err := clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Delete(context.Background(), mutationWebhook.Name, metav1.DeleteOptions{}); err != nil {
clusterWatcher.Log.Warnf("cannot delete mutation webhook %s, error=%s", mutationWebhook.Name, err.Error())
}
// == ROTATE ==
// update controller tls secret
controllerSecret := deployments.GetKubeArmorControllerTLSSecret(common.Namespace, caCert.String(), tlsCrt.String(), tlsKey.String())
controllerSecret = addOwnership(controllerSecret).(*corev1.Secret)
if _, err := clusterWatcher.Client.CoreV1().Secrets(common.Namespace).Update(context.Background(), controllerSecret, metav1.UpdateOptions{}); err != nil {
clusterWatcher.Log.Warnf("cannot update controller tls secret %s, error=%s", controllerSecret.Name, err.Error())
}
// == ROLLOUT ==
// create mutation webhook configuration
if _, err := clusterWatcher.Client.AdmissionregistrationV1().MutatingWebhookConfigurations().Create(context.Background(), mutationWebhook, metav1.CreateOptions{}); err != nil {
clusterWatcher.Log.Warnf("Cannot create mutation webhook %s, error=%s", mutationWebhook.Name, err.Error())
}
// scale up controller deployment to previous settings
controllerDeployment.Spec.Replicas = replicas
if _, err := clusterWatcher.Client.AppsV1().Deployments(common.Namespace).Update(context.Background(), controllerDeployment, metav1.UpdateOptions{}); err != nil {
clusterWatcher.Log.Warnf("cannot scale down controller %s, error=%s", controllerDeployment.Name, err.Error())
}
clusterWatcher.Log.Info("Tls rotation completed")
}
Loading