Inject preStop hook into the proxy sidecar container to stop it last

bin/_docker.sh

@@ -5,7 +5,7 @@ bindir=$( cd "${BASH_SOURCE[0]%/*}" && pwd )
 . "$bindir"/_log.sh
 
 # TODO this should be set to the canonical public docker registry; we can override this
-# docker regsistry in, for instance, CI.
+# docker registry in, for instance, CI.
 export DOCKER_REGISTRY=${DOCKER_REGISTRY:-gcr.io/linkerd-io}
 
 # When set, causes docker's build output to be emitted to stderr.

charts/linkerd2/README.md

@@ -126,6 +126,7 @@ The following table lists the configurable parameters of the Linkerd2 chart and
 | `proxy.trace.collectorSvcAccount`     | Service account associated with the Trace collector instance                                                                                                                                                                ||
 | `proxy.trace.collectorSvcAddr`        | Collector Service address for the proxies to send Trace Data                                                                                                                                                                ||
 | `proxy.uid`                           | User id under which the proxy runs                                                                                                                                                    | `2102`                               |
+| `proxy.waitBeforeExitSeconds`         | The proxy sidecar will stay alive for at least the given period before receiving SIGTERM signal from Kubernetes but no longer than pod's `terminationGracePeriodSeconds`.             | `0`                                  |
 | `proxyInit.ignoreInboundPorts`        | Inbound ports the proxy should ignore                                                                                                                                                                                       ||
 | `proxyInit.ignoreOutboundPorts`       | Outbound ports the proxy should ignore                                                                                                                                                                                      ||
 | `proxyInit.image.name`                | Docker image for the proxy-init container                                                                                                                                             | `gcr.io/linkerd-io/proxy-init`       |

charts/linkerd2/values.yaml

@@ -87,6 +87,11 @@ proxy:
     collectorSvcAddr: ""
     collectorSvcAccount: default
   uid: 2102
+  # If set, the proxy's pre-stop hook will postpone the Kubernetes's SIGTERM signal
+  # and wait for this duration before letting the proxy process the SIGTERM signal.
+  # See https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks
+  # for more info on container lifecycle hooks.
+  waitBeforeExitSeconds: 0
 
 # proxy-init configuration
 proxyInit:

charts/partials/templates/_proxy.tpl

@@ -110,6 +110,15 @@ securityContext:
   readOnlyRootFilesystem: true
   runAsUser: {{.Values.proxy.uid}}
 terminationMessagePolicy: FallbackToLogsOnError
+{{- if .Values.proxy.waitBeforeExitSeconds }}
+lifecycle:
+  preStop:
+    exec:
+      command:
+        - /bin/bash
+        - -c
+        - sleep {{.Values.proxy.waitBeforeExitSeconds}}
+{{- end }}
 {{- if or (not .Values.proxy.disableIdentity) (.Values.proxy.saMountPath) }}
 volumeMounts:
 {{- if not .Values.proxy.disableIdentity }}

cli/cmd/doc.go

@@ -207,5 +207,9 @@ func generateAnnotationsDocs() []annotationDoc {
 			Name:        k8s.ProxyTraceCollectorSvcAccountAnnotation,
 			Description: "The trace collector's service account name. E.g., `tracing-service-account`. If not provided, it will be defaulted to `default`.",
 		},
+		{
+			Name:        k8s.ProxyWaitBeforeExitSecondsAnnotation,
+			Description: "The proxy sidecar will stay alive for at least the given period before receiving SIGTERM signal from Kubernetes but no longer than pod's `terminationGracePeriodSeconds`. If not provided, it will be defaulted to `0`",
+		},
 	}
 }

cli/cmd/inject.go

@@ -100,6 +100,11 @@ sub-folders, or coming from stdin.`,
 		&manualOption, "manual", manualOption,
 		"Include the proxy sidecar container spec in the YAML output (the auto-injector won't pick it up, so config annotations aren't supported) (default false)",
 	)
+	flags.Uint64Var(
+		&options.waitBeforeExitSeconds, "wait-before-exit-seconds", options.waitBeforeExitSeconds,
+		"The period during which the proxy sidecar must stay alive while its pod is terminating. "+
+			"Must be smaller than terminationGracePeriodSeconds for the pod (default 0)",
+	)
 	flags.BoolVar(
 		&options.disableIdentity, "disable-identity", options.disableIdentity,
 		"Disables resources from participating in TLS identity",
@@ -439,6 +444,13 @@ func (options *proxyConfigOptions) overrideConfigs(configs *cfg.All, overrideAnn
 	if options.traceCollectorSvcAccount != "" {
 		overrideAnnotations[k8s.ProxyTraceCollectorSvcAccountAnnotation] = options.traceCollectorSvcAccount
 	}
+	if options.waitBeforeExitSeconds != 0 {
+		overrideAnnotations[k8s.ProxyWaitBeforeExitSecondsAnnotation] = uintToString(options.waitBeforeExitSeconds)
+	}
+}
+
+func uintToString(v uint64) string {
+	return strconv.FormatUint(v, 10)
 }
 
 func toPort(p uint) *cfg.Port {

cli/cmd/install.go

@@ -198,6 +198,7 @@ func newInstallOptionsWithDefaults() (*installOptions, error) {
 			proxyCPULimit:          defaults.Proxy.Resources.CPU.Limit,
 			proxyMemoryLimit:       defaults.Proxy.Resources.Memory.Limit,
 			enableExternalProfiles: defaults.Proxy.EnableExternalProfiles,
+			waitBeforeExitSeconds:  defaults.Proxy.WaitBeforeExitSeconds,
 		},
 		identityOptions: &installIdentityOptions{
 			trustDomain:            defaults.Identity.TrustDomain,

cli/cmd/root.go

@@ -199,6 +199,7 @@ type proxyConfigOptions struct {
 	enableExternalProfiles   bool
 	traceCollector           string
 	traceCollectorSvcAccount string
+	waitBeforeExitSeconds    uint64
 	// ignoreCluster is not validated by validate().
 	ignoreCluster   bool
 	disableIdentity bool

pkg/charts/linkerd2/values.go

@@ -95,6 +95,7 @@ type (
 		Resources              *Resources    `json:"resources"`
 		Trace                  *Trace        `json:"trace"`
 		UID                    int64         `json:"uid"`
+		WaitBeforeExitSeconds  uint64        `json:"waitBeforeExitSeconds"`
 	}
 
 	// ProxyInit contains the fields to set the proxy-init container

pkg/charts/linkerd2/values_test.go

@@ -98,7 +98,8 @@ func TestNewValues(t *testing.T) {
 				CollectorSvcAddr:    "",
 				CollectorSvcAccount: "default",
 			},
-			UID: 2102,
+			UID:                   2102,
+			WaitBeforeExitSeconds: 0,
 		},
 
 		ProxyInit: &ProxyInit{

pkg/inject/inject.go

@@ -467,8 +467,9 @@ func (conf *ResourceConfig) injectPodSpec(values *patch) {
 			Inbound:  conf.proxyInboundPort(),
 			Outbound: conf.proxyOutboundPort(),
 		},
-		UID:       conf.proxyUID(),
-		Resources: conf.proxyResourceRequirements(),
+		UID:                   conf.proxyUID(),
+		Resources:             conf.proxyResourceRequirements(),
+		WaitBeforeExitSeconds: conf.proxyWaitBeforeExitSeconds(),
 	}
 
 	if v := conf.pod.meta.Annotations[k8s.ProxyEnableDebugAnnotation]; v != "" {
@@ -757,6 +758,20 @@ func (conf *ResourceConfig) tapDisabled() bool {
 	return false
 }
 
+func (conf *ResourceConfig) proxyWaitBeforeExitSeconds() uint64 {
+	if override := conf.getOverride(k8s.ProxyWaitBeforeExitSecondsAnnotation); override != "" {
+		waitBeforeExitSeconds, err := strconv.ParseUint(override, 10, 64)
+		if nil != err {
+			log.Warnf("unrecognized value used for the %s annotation, uint64 is expected: %s",
+				k8s.ProxyWaitBeforeExitSecondsAnnotation, override)
+		}
+
+		return waitBeforeExitSeconds
+	}
+
+	return 0
+}
+
 func (conf *ResourceConfig) proxyResourceRequirements() *l5dcharts.Resources {
 	var (
 		requestCPU    k8sResource.Quantity

pkg/inject/inject_test.go

@@ -15,23 +15,24 @@ import (
 )
 
 type expectedProxyConfigs struct {
-	identityContext      *config.IdentityContext
-	image                string
-	imagePullPolicy      string
-	proxyVersion         string
-	controlPort          int32
-	inboundPort          int32
-	adminPort            int32
-	outboundPort         int32
-	logLevel             string
-	resourceRequirements *l5dcharts.Resources
-	proxyUID             int64
-	initImage            string
-	initImagePullPolicy  string
-	initVersion          string
-	inboundSkipPorts     string
-	outboundSkipPorts    string
-	trace                *l5dcharts.Trace
+	identityContext            *config.IdentityContext
+	image                      string
+	imagePullPolicy            string
+	proxyVersion               string
+	controlPort                int32
+	inboundPort                int32
+	adminPort                  int32
+	outboundPort               int32
+	proxyWaitBeforeExitSeconds uint64
+	logLevel                   string
+	resourceRequirements       *l5dcharts.Resources
+	proxyUID                   int64
+	initImage                  string
+	initImagePullPolicy        string
+	initVersion                string
+	inboundSkipPorts           string
+	outboundSkipPorts          string
+	trace                      *l5dcharts.Trace
 }
 
 func TestConfigAccessors(t *testing.T) {
@@ -107,20 +108,22 @@ func TestConfigAccessors(t *testing.T) {
 							k8s.ProxyVersionOverrideAnnotation:          proxyVersionOverride,
 							k8s.ProxyTraceCollectorSvcAddrAnnotation:    "oc-collector.tracing:55678",
 							k8s.ProxyTraceCollectorSvcAccountAnnotation: "default",
+							k8s.ProxyWaitBeforeExitSecondsAnnotation:    "123",
 						},
 					},
 					Spec: corev1.PodSpec{},
 				},
 			},
 			expected: expectedProxyConfigs{
-				image:           "gcr.io/linkerd-io/proxy",
-				imagePullPolicy: "Always",
-				proxyVersion:    proxyVersionOverride,
-				controlPort:     int32(4000),
-				inboundPort:     int32(5000),
-				adminPort:       int32(5001),
-				outboundPort:    int32(5002),
-				logLevel:        "debug,linkerd2_proxy=debug",
+				image:                      "gcr.io/linkerd-io/proxy",
+				imagePullPolicy:            "Always",
+				proxyVersion:               proxyVersionOverride,
+				controlPort:                int32(4000),
+				inboundPort:                int32(5000),
+				adminPort:                  int32(5001),
+				outboundPort:               int32(5002),
+				proxyWaitBeforeExitSeconds: 123,
+				logLevel:                   "debug,linkerd2_proxy=debug",
 				resourceRequirements: &l5dcharts.Resources{
 					CPU: l5dcharts.Constraints{
 						Limit:   "1500m",
@@ -151,15 +154,16 @@ func TestConfigAccessors(t *testing.T) {
 				},
 			},
 			expected: expectedProxyConfigs{
-				identityContext: &config.IdentityContext{},
-				image:           "gcr.io/linkerd-io/proxy",
-				imagePullPolicy: "IfNotPresent",
-				proxyVersion:    proxyVersion,
-				controlPort:     int32(9000),
-				inboundPort:     int32(6000),
-				adminPort:       int32(6001),
-				outboundPort:    int32(6002),
-				logLevel:        "info,linkerd2_proxy=debug",
+				identityContext:            &config.IdentityContext{},
+				image:                      "gcr.io/linkerd-io/proxy",
+				imagePullPolicy:            "IfNotPresent",
+				proxyVersion:               proxyVersion,
+				controlPort:                int32(9000),
+				inboundPort:                int32(6000),
+				adminPort:                  int32(6001),
+				outboundPort:               int32(6002),
+				proxyWaitBeforeExitSeconds: 0,
+				logLevel:                   "info,linkerd2_proxy=debug",
 				resourceRequirements: &l5dcharts.Resources{
 					CPU: l5dcharts.Constraints{
 						Limit:   "1",
@@ -200,21 +204,23 @@ func TestConfigAccessors(t *testing.T) {
 				k8s.ProxyVersionOverrideAnnotation:          proxyVersionOverride,
 				k8s.ProxyTraceCollectorSvcAddrAnnotation:    "oc-collector.tracing:55678",
 				k8s.ProxyTraceCollectorSvcAccountAnnotation: "default",
+				k8s.ProxyWaitBeforeExitSecondsAnnotation:    "123",
 			},
 			spec: appsv1.DeploymentSpec{
 				Template: corev1.PodTemplateSpec{
 					Spec: corev1.PodSpec{},
 				},
 			},
 			expected: expectedProxyConfigs{
-				image:           "gcr.io/linkerd-io/proxy",
-				imagePullPolicy: "Always",
-				proxyVersion:    proxyVersionOverride,
-				controlPort:     int32(4000),
-				inboundPort:     int32(5000),
-				adminPort:       int32(5001),
-				outboundPort:    int32(5002),
-				logLevel:        "debug,linkerd2_proxy=debug",
+				image:                      "gcr.io/linkerd-io/proxy",
+				imagePullPolicy:            "Always",
+				proxyVersion:               proxyVersionOverride,
+				controlPort:                int32(4000),
+				inboundPort:                int32(5000),
+				adminPort:                  int32(5001),
+				outboundPort:               int32(5002),
+				proxyWaitBeforeExitSeconds: 123,
+				logLevel:                   "debug,linkerd2_proxy=debug",
 				resourceRequirements: &l5dcharts.Resources{
 					CPU: l5dcharts.Constraints{
 						Limit:   "1500m",
@@ -237,6 +243,45 @@ func TestConfigAccessors(t *testing.T) {
 				},
 			},
 		},
+		{id: "use not a uint value for ProxyWaitBeforeExitSecondsAnnotation annotation",
+			nsAnnotations: map[string]string{
+				k8s.ProxyWaitBeforeExitSecondsAnnotation: "-111",
+			},
+			spec: appsv1.DeploymentSpec{
+				Template: corev1.PodTemplateSpec{
+					ObjectMeta: metav1.ObjectMeta{},
+					Spec:       corev1.PodSpec{},
+				},
+			},
+			expected: expectedProxyConfigs{
+				identityContext:            &config.IdentityContext{},
+				image:                      "gcr.io/linkerd-io/proxy",
+				imagePullPolicy:            "IfNotPresent",
+				proxyVersion:               proxyVersion,
+				controlPort:                int32(9000),
+				inboundPort:                int32(6000),
+				adminPort:                  int32(6001),
+				outboundPort:               int32(6002),
+				proxyWaitBeforeExitSeconds: 0,
+				logLevel:                   "info,linkerd2_proxy=debug",
+				resourceRequirements: &l5dcharts.Resources{
+					CPU: l5dcharts.Constraints{
+						Limit:   "1",
+						Request: "200m",
+					},
+					Memory: l5dcharts.Constraints{
+						Limit:   "128",
+						Request: "64",
+					},
+				},
+				proxyUID:            int64(8888),
+				initImage:           "gcr.io/linkerd-io/proxy-init",
+				initImagePullPolicy: "IfNotPresent",
+				initVersion:         version.ProxyInitVersion,
+				inboundSkipPorts:    "53",
+				outboundSkipPorts:   "9079",
+			},
+		},
 	}
 
 	for _, tc := range testCases {
@@ -315,6 +360,13 @@ func TestConfigAccessors(t *testing.T) {
 				}
 			})
 
+			t.Run("proxyWaitBeforeExitSeconds", func(t *testing.T) {
+				expected := testCase.expected.proxyWaitBeforeExitSeconds
+				if actual := resourceConfig.proxyWaitBeforeExitSeconds(); expected != actual {
+					t.Errorf("Expected: %v Actual: %v", expected, actual)
+				}
+			})
+
 			t.Run("proxyLogLevel", func(t *testing.T) {
 				expected := testCase.expected.logLevel
 				if actual := resourceConfig.proxyLogLevel(); expected != actual {

pkg/k8s/labels.go

@@ -102,6 +102,9 @@ const (
 	// ProxyConfigAnnotationsPrefix is the prefix of all config-related annotations
 	ProxyConfigAnnotationsPrefix = "config.linkerd.io"
 
+	// ProxyConfigAnnotationsPrefixAlpha is the prefix of newly released config-related annotations
+	ProxyConfigAnnotationsPrefixAlpha = "config.alpha.linkerd.io"
+
 	// ProxyImageAnnotation can be used to override the proxyImage config.
 	ProxyImageAnnotation = ProxyConfigAnnotationsPrefix + "/proxy-image"
 
@@ -178,10 +181,15 @@ const (
 	// its value.
 	ProxyTraceCollectorSvcAddrAnnotation = ProxyConfigAnnotationsPrefix + "/trace-collector"
 
+	// ProxyWaitBeforeExitSecondsAnnotation makes the proxy container to wait for the given period before exiting
+	// after the Pod entered the Terminating state. Must be smaller than terminationGracePeriodSeconds
+	// configured for the Pod
+	ProxyWaitBeforeExitSecondsAnnotation = ProxyConfigAnnotationsPrefixAlpha + "/proxy-wait-before-exit-seconds"
+
 	// ProxyTraceCollectorSvcAccountAnnotation is used to specify the service account
 	// associated with the trace collector. It is used to create the service's
 	// mTLS identity.
-	ProxyTraceCollectorSvcAccountAnnotation = "config.alpha.linkerd.io/trace-collector-service-account"
+	ProxyTraceCollectorSvcAccountAnnotation = ProxyConfigAnnotationsPrefixAlpha + "/trace-collector-service-account"
 
 	// IdentityModeDefault is assigned to IdentityModeAnnotation to
 	// use the control plane's default identity scheme.