Skip to content

Commit

Permalink
feature reload-on-delete implemented, test cases enhanced
Browse files Browse the repository at this point in the history
  • Loading branch information
deschmih committed Jun 18, 2024
1 parent 8595b4a commit 2260d72
Show file tree
Hide file tree
Showing 12 changed files with 568 additions and 189 deletions.
7 changes: 6 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,7 @@ helm uninstall {{RELEASE_NAME}} -n {{NAMESPACE}}
| `reloader.ignoreSecrets` | To ignore secrets. Valid value are either `true` or `false`. Either `ignoreSecrets` or `ignoreConfigMaps` can be ignored, not both at the same time | boolean | `false` |
| `reloader.ignoreConfigMaps` | To ignore configMaps. Valid value are either `true` or `false` | boolean | `false` |
| `reloader.reloadOnCreate` | Enable reload on create events. Valid value are either `true` or `false` | boolean | `false` |
| `reloader.reloadOnDelete` | Enable reload on delete events. Valid value are either `true` or `false` | boolean | `false` |
| `reloader.syncAfterRestart` | Enable sync after Reloader restarts for **Add** events, works only when reloadOnCreate is `true`. Valid value are either `true` or `false` | boolean | `false` |
| `reloader.reloadStrategy` | Strategy to trigger resource restart, set to either `default`, `env-vars` or `annotations` | enumeration | `default` |
| `reloader.ignoreNamespaces` | List of comma separated namespaces to ignore, if multiple are provided, they are combined with the AND operator | string | `""` |
Expand Down Expand Up @@ -377,11 +378,15 @@ helm uninstall {{RELEASE_NAME}} -n {{NAMESPACE}}
1. Configmaps/secrets being added to the cache will cause Reloader to perform a rolling update of the associated workload
1. When applications are deployed for the first time, Reloader will perform a rolling update of the associated workload
1. If you are running Reloader in HA mode all workloads will have a rolling update performed when a new leader is elected
- `reloadOnDelete` controls how Reloader handles secrets being deleted. If `reloadOnDelete` is set to true:
1. Configmaps/secrets being deleted will cause Reloader to perform a rolling update of the associated workload
- `serviceMonitor` will be removed in future releases of Reloader in favour of Pod monitor
- If `reloadOnCreate` is set to false:
1. Updates to configmaps/secrets that occur while there is no leader will not be picked up by the new leader until a subsequent update of the configmap/secret occurs
1. In the worst case the window in which there can be no leader is 15s as this is the LeaseDuration
- By default, `reloadOnCreate` and `syncAfterRestart` are both set to false. Both need to be enabled explicitly
- If `reloadOnDelete` is set to false:
1. Deleting of configmaps/secrets has no effect to pods that references these resources.
- By default, `reloadOnCreate`, `reloadOnDelete` and `syncAfterRestart` are all set to false. All need to be enabled explicitly

## Help

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ spec:
{{- . | toYaml | nindent 10 }}
{{- end }}
{{- end }}
{{- if or (.Values.reloader.logFormat) (.Values.reloader.ignoreSecrets) (.Values.reloader.ignoreNamespaces) (.Values.reloader.namespaceSelector) (.Values.reloader.resourceLabelSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA) (.Values.reloader.autoReloadAll)}}
{{- if or (.Values.reloader.logFormat) (.Values.reloader.ignoreSecrets) (.Values.reloader.ignoreNamespaces) (.Values.reloader.namespaceSelector) (.Values.reloader.resourceLabelSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (eq .Values.reloader.reloadOnDelete true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA) (.Values.reloader.autoReloadAll)}}
args:
{{- if .Values.reloader.logFormat }}
- "--log-format={{ .Values.reloader.logFormat }}"
Expand Down Expand Up @@ -231,6 +231,9 @@ spec:
{{- if eq .Values.reloader.reloadOnCreate true }}
- "--reload-on-create={{ .Values.reloader.reloadOnCreate }}"
{{- end }}
{{- if eq .Values.reloader.reloadOnDelete true }}
- "--reload-on-delete={{ .Values.reloader.reloadOnDelete }}"
{{- end }}
{{- if eq .Values.reloader.syncAfterRestart true }}
- "--sync-after-restart={{ .Values.reloader.syncAfterRestart }}"
{{- end }}
Expand Down
1 change: 1 addition & 0 deletions deployments/kubernetes/chart/reloader/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ reloader:
ignoreSecrets: false
ignoreConfigMaps: false
reloadOnCreate: false
reloadOnDelete: false
syncAfterRestart: false
reloadStrategy: default # Set to default, env-vars or annotations
ignoreNamespaces: "" # Comma separated list of namespaces to ignore
Expand Down
1 change: 1 addition & 0 deletions internal/pkg/cmd/reloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ func NewReloaderCommand() *cobra.Command {
cmd.PersistentFlags().StringVar(&options.IsArgoRollouts, "is-Argo-Rollouts", "false", "Add support for argo rollouts")
cmd.PersistentFlags().StringVar(&options.ReloadStrategy, constants.ReloadStrategyFlag, constants.EnvVarsReloadStrategy, "Specifies the desired reload strategy")
cmd.PersistentFlags().StringVar(&options.ReloadOnCreate, "reload-on-create", "false", "Add support to watch create events")
cmd.PersistentFlags().StringVar(&options.ReloadOnDelete, "reload-on-delete", "false", "Add support to watch delete events")
cmd.PersistentFlags().BoolVar(&options.EnableHA, "enable-ha", false, "Adds support for running multiple replicas via leadership election")
cmd.PersistentFlags().BoolVar(&options.SyncAfterRestart, "sync-after-restart", false, "Sync add events after reloader restarts")

Expand Down
13 changes: 11 additions & 2 deletions internal/pkg/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -178,13 +178,22 @@ func (c *Controller) Update(old interface{}, new interface{}) {

// Delete function to add an object to the queue in case of deleting a resource
func (c *Controller) Delete(old interface{}) {

if options.ReloadOnDelete == "true" {
if !c.resourceInIgnoredNamespace(old) && c.resourceInSelectedNamespaces(old) && secretControllerInitialized && configmapControllerInitialized {
c.queue.Add(handler.ResourceDeleteHandler{
Resource: old,
Collectors: c.collectors,
Recorder: c.recorder,
})
}
}

switch object := old.(type) {
case *v1.Namespace:
c.removeSelectedNamespaceFromCache(*object)
return
}

// Todo: Any future delete event can be handled here
}

// Run function for controller which handles the queue
Expand Down
2 changes: 1 addition & 1 deletion internal/pkg/handler/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ func (r ResourceCreatedHandler) Handle() error {
return sendUpgradeWebhook(config, options.WebhookUrl)
}
// process resource based on its type
return doRollingUpgrade(config, r.Collectors, r.Recorder)
return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy)
}
return nil
}
Expand Down
92 changes: 92 additions & 0 deletions internal/pkg/handler/delete.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package handler

import (
"github.com/sirupsen/logrus"
"github.com/stakater/Reloader/internal/pkg/callbacks"
"github.com/stakater/Reloader/internal/pkg/constants"
"github.com/stakater/Reloader/internal/pkg/metrics"
"github.com/stakater/Reloader/internal/pkg/options"
"github.com/stakater/Reloader/internal/pkg/testutil"
"github.com/stakater/Reloader/internal/pkg/util"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/record"
)

// ResourceDeleteHandler contains new objects
type ResourceDeleteHandler struct {
Resource interface{}
Collectors metrics.Collectors
Recorder record.EventRecorder
}

// Handle processes resources being deleted
func (r ResourceDeleteHandler) Handle() error {
if r.Resource == nil {
logrus.Errorf("Resource delete handler received nil resource")
} else {
config, _ := r.GetConfig()
// Send webhook
if options.WebhookUrl != "" {
return sendUpgradeWebhook(config, options.WebhookUrl)
}
// process resource based on its type
return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeDeleteStrategy)
}
return nil
}

// GetConfig gets configurations containing SHA, annotations, namespace and resource name
func (r ResourceDeleteHandler) GetConfig() (util.Config, string) {
var oldSHAData string
var config util.Config
if _, ok := r.Resource.(*v1.ConfigMap); ok {
config = util.GetConfigmapConfig(r.Resource.(*v1.ConfigMap))
} else if _, ok := r.Resource.(*v1.Secret); ok {
config = util.GetSecretConfig(r.Resource.(*v1.Secret))
} else {
logrus.Warnf("Invalid resource: Resource should be 'Secret' or 'Configmap' but found, %v", r.Resource)
}
return config, oldSHAData
}

func invokeDeleteStrategy(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
if options.ReloadStrategy == constants.AnnotationsReloadStrategy {
return removePodAnnotations(upgradeFuncs, item, config, autoReload)
}

return removeContainerEnvVars(upgradeFuncs, item, config, autoReload)
}

func removePodAnnotations(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
config.SHAValue = testutil.GetSHAfromEmptyData()
return updatePodAnnotations(upgradeFuncs, item, config, autoReload)
}

func removeContainerEnvVars(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
envVar := getEnvVarName(config.ResourceName, config.Type)
container := getContainerUsingResource(upgradeFuncs, item, config, autoReload)

if container == nil {
return constants.NoContainerFound
}

//remove if env var exists
containers := upgradeFuncs.ContainersFunc(item)
for i := range containers {
envs := containers[i].Env
index := -1
for j := range envs {
if envs[j].Name == envVar {
index = j
break
}
}
if index != -1 {
containers[i].Env = append(containers[i].Env[:index], containers[i].Env[index+1:]...)
return constants.Updated
}
}

return constants.NotUpdated
}
2 changes: 1 addition & 1 deletion internal/pkg/handler/update.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ func (r ResourceUpdatedHandler) Handle() error {
return sendUpgradeWebhook(config, options.WebhookUrl)
}
// process resource based on its type
return doRollingUpgrade(config, r.Collectors, r.Recorder)
return doRollingUpgrade(config, r.Collectors, r.Recorder, invokeReloadStrategy)
}
}
return nil
Expand Down
48 changes: 29 additions & 19 deletions internal/pkg/handler/upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,35 +142,35 @@ func sendWebhook(url string) (string, []error) {
return buffer.String(), nil
}

func doRollingUpgrade(config util.Config, collectors metrics.Collectors, recorder record.EventRecorder) error {
func doRollingUpgrade(config util.Config, collectors metrics.Collectors, recorder record.EventRecorder, invoke invokeStrategy) error {
clients := kube.GetClients()

err := rollingUpgrade(clients, config, GetDeploymentRollingUpgradeFuncs(), collectors, recorder)
err := rollingUpgrade(clients, config, GetDeploymentRollingUpgradeFuncs(), collectors, recorder, invoke)
if err != nil {
return err
}
err = rollingUpgrade(clients, config, GetCronJobCreateJobFuncs(), collectors, recorder)
err = rollingUpgrade(clients, config, GetCronJobCreateJobFuncs(), collectors, recorder, invoke)
if err != nil {
return err
}
err = rollingUpgrade(clients, config, GetDaemonSetRollingUpgradeFuncs(), collectors, recorder)
err = rollingUpgrade(clients, config, GetDaemonSetRollingUpgradeFuncs(), collectors, recorder, invoke)
if err != nil {
return err
}
err = rollingUpgrade(clients, config, GetStatefulSetRollingUpgradeFuncs(), collectors, recorder)
err = rollingUpgrade(clients, config, GetStatefulSetRollingUpgradeFuncs(), collectors, recorder, invoke)
if err != nil {
return err
}

if kube.IsOpenshift {
err = rollingUpgrade(clients, config, GetDeploymentConfigRollingUpgradeFuncs(), collectors, recorder)
err = rollingUpgrade(clients, config, GetDeploymentConfigRollingUpgradeFuncs(), collectors, recorder, invoke)
if err != nil {
return err
}
}

if options.IsArgoRollouts == "true" {
err = rollingUpgrade(clients, config, GetArgoRolloutRollingUpgradeFuncs(), collectors, recorder)
err = rollingUpgrade(clients, config, GetArgoRolloutRollingUpgradeFuncs(), collectors, recorder, invoke)
if err != nil {
return err
}
Expand All @@ -179,17 +179,17 @@ func doRollingUpgrade(config util.Config, collectors metrics.Collectors, recorde
return nil
}

func rollingUpgrade(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder) error {
func rollingUpgrade(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy) error {

err := PerformRollingUpgrade(clients, config, upgradeFuncs, collectors, recorder)
err := PerformAction(clients, config, upgradeFuncs, collectors, recorder, strategy)
if err != nil {
logrus.Errorf("Rolling upgrade for '%s' failed with error = %v", config.ResourceName, err)
}
return err
}

// PerformRollingUpgrade upgrades the deployment if there is any change in configmap or secret data
func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder) error {
// PerformAction invokes the deployment if there is any change in configmap or secret data
func PerformAction(clients kube.Clients, config util.Config, upgradeFuncs callbacks.RollingUpgradeFuncs, collectors metrics.Collectors, recorder record.EventRecorder, strategy invokeStrategy) error {
items := upgradeFuncs.ItemsFunc(clients, config.Namespace)

for _, i := range items {
Expand All @@ -210,7 +210,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
reloaderEnabled, _ := strconv.ParseBool(reloaderEnabledValue)
typedAutoAnnotationEnabled, _ := strconv.ParseBool(typedAutoAnnotationEnabledValue)
if reloaderEnabled || typedAutoAnnotationEnabled || reloaderEnabledValue == "" && typedAutoAnnotationEnabledValue == "" && options.AutoReloadAll {
result = invokeReloadStrategy(upgradeFuncs, i, config, true)
result = strategy(upgradeFuncs, i, config, true)
}

if result != constants.Updated && annotationValue != "" {
Expand All @@ -219,7 +219,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
value = strings.TrimSpace(value)
re := regexp.MustCompile("^" + value + "$")
if re.Match([]byte(config.ResourceName)) {
result = invokeReloadStrategy(upgradeFuncs, i, config, false)
result = strategy(upgradeFuncs, i, config, false)
if result == constants.Updated {
break
}
Expand All @@ -230,7 +230,7 @@ func PerformRollingUpgrade(clients kube.Clients, config util.Config, upgradeFunc
if result != constants.Updated && searchAnnotationValue == "true" {
matchAnnotationValue := config.ResourceAnnotations[options.SearchMatchAnnotation]
if matchAnnotationValue == "true" {
result = invokeReloadStrategy(upgradeFuncs, i, config, true)
result = strategy(upgradeFuncs, i, config, true)
}
}

Expand Down Expand Up @@ -380,6 +380,8 @@ func getContainerUsingResource(upgradeFuncs callbacks.RollingUpgradeFuncs, item
return container
}

type invokeStrategy func(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result

func invokeReloadStrategy(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
if options.ReloadStrategy == constants.AnnotationsReloadStrategy {
return updatePodAnnotations(upgradeFuncs, item, config, autoReload)
Expand Down Expand Up @@ -416,6 +418,13 @@ func updatePodAnnotations(upgradeFuncs callbacks.RollingUpgradeFuncs, item runti
return constants.Updated
}

func getReloaderAnnotationKey() string {
return fmt.Sprintf("%s/%s",
constants.ReloaderAnnotationPrefix,
constants.LastReloadedFromAnnotation,
)
}

func createReloadedAnnotations(target *util.ReloadSource) (map[string]string, error) {
if target == nil {
return nil, errors.New("target is required")
Expand All @@ -426,10 +435,7 @@ func createReloadedAnnotations(target *util.ReloadSource) (map[string]string, er
// Intentionally only storing the last item in order to keep
// the generated annotations as small as possible.
annotations := make(map[string]string)
lastReloadedResourceName := fmt.Sprintf("%s/%s",
constants.ReloaderAnnotationPrefix,
constants.LastReloadedFromAnnotation,
)
lastReloadedResourceName := getReloaderAnnotationKey()

lastReloadedResource, err := json.Marshal(target)
if err != nil {
Expand All @@ -440,9 +446,13 @@ func createReloadedAnnotations(target *util.ReloadSource) (map[string]string, er
return annotations, nil
}

func getEnvVarName(resourceName string, typeName string) string {
return constants.EnvVarPrefix + util.ConvertToEnvVarName(resourceName) + "_" + typeName
}

func updateContainerEnvVars(upgradeFuncs callbacks.RollingUpgradeFuncs, item runtime.Object, config util.Config, autoReload bool) constants.Result {
var result constants.Result
envVar := constants.EnvVarPrefix + util.ConvertToEnvVarName(config.ResourceName) + "_" + config.Type
envVar := getEnvVarName(config.ResourceName, config.Type)
container := getContainerUsingResource(upgradeFuncs, item, config, autoReload)

if container == nil {
Expand Down
Loading

0 comments on commit 2260d72

Please sign in to comment.