diff --git a/bin/_docker.sh b/bin/_docker.sh index 9480f4c293616..f0ae57e5853bc 100644 --- a/bin/_docker.sh +++ b/bin/_docker.sh @@ -5,7 +5,7 @@ bindir=$( cd "${BASH_SOURCE[0]%/*}" && pwd ) . "$bindir"/_log.sh # TODO this should be set to the canonical public docker registry; we can override this -# docker regsistry in, for instance, CI. +# docker registry in, for instance, CI. export DOCKER_REGISTRY=${DOCKER_REGISTRY:-gcr.io/linkerd-io} # When set, causes docker's build output to be emitted to stderr. diff --git a/charts/linkerd2/README.md b/charts/linkerd2/README.md index da24ea2bfcaa3..3d5e4675a5f19 100644 --- a/charts/linkerd2/README.md +++ b/charts/linkerd2/README.md @@ -126,6 +126,7 @@ The following table lists the configurable parameters of the Linkerd2 chart and | `proxy.trace.collectorSvcAccount` | Service account associated with the Trace collector instance || | `proxy.trace.collectorSvcAddr` | Collector Service address for the proxies to send Trace Data || | `proxy.uid` | User id under which the proxy runs | `2102` | +| `proxy.waitBeforeExitSeconds` | The proxy sidecar will stay alive for at least the given period before receiving SIGTERM signal from Kubernetes but no longer than pod's `terminationGracePeriodSeconds`. | `0` | | `proxyInit.ignoreInboundPorts` | Inbound ports the proxy should ignore || | `proxyInit.ignoreOutboundPorts` | Outbound ports the proxy should ignore || | `proxyInit.image.name` | Docker image for the proxy-init container | `gcr.io/linkerd-io/proxy-init` | diff --git a/charts/linkerd2/values.yaml b/charts/linkerd2/values.yaml index 8666526405bd2..2cd50cdbcc262 100644 --- a/charts/linkerd2/values.yaml +++ b/charts/linkerd2/values.yaml @@ -87,6 +87,11 @@ proxy: collectorSvcAddr: "" collectorSvcAccount: default uid: 2102 + # If set, the proxy's pre-stop hook will postpone the Kubernetes's SIGTERM signal + # and wait for this duration before letting the proxy process the SIGTERM signal. + # See https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks + # for more info on container lifecycle hooks. + waitBeforeExitSeconds: 0 # proxy-init configuration proxyInit: diff --git a/charts/partials/templates/_proxy.tpl b/charts/partials/templates/_proxy.tpl index a38a5830be5be..3d4e51cf53fd0 100644 --- a/charts/partials/templates/_proxy.tpl +++ b/charts/partials/templates/_proxy.tpl @@ -110,6 +110,15 @@ securityContext: readOnlyRootFilesystem: true runAsUser: {{.Values.proxy.uid}} terminationMessagePolicy: FallbackToLogsOnError +{{- if .Values.proxy.waitBeforeExitSeconds }} +lifecycle: + preStop: + exec: + command: + - /bin/bash + - -c + - sleep {{.Values.proxy.waitBeforeExitSeconds}} +{{- end }} {{- if or (not .Values.proxy.disableIdentity) (.Values.proxy.saMountPath) }} volumeMounts: {{- if not .Values.proxy.disableIdentity }} diff --git a/cli/cmd/doc.go b/cli/cmd/doc.go index 230bb8b60ba51..ac1c7d72ae106 100644 --- a/cli/cmd/doc.go +++ b/cli/cmd/doc.go @@ -207,5 +207,9 @@ func generateAnnotationsDocs() []annotationDoc { Name: k8s.ProxyTraceCollectorSvcAccountAnnotation, Description: "The trace collector's service account name. E.g., `tracing-service-account`. If not provided, it will be defaulted to `default`.", }, + { + Name: k8s.ProxyWaitBeforeExitSecondsAnnotation, + Description: "The proxy sidecar will stay alive for at least the given period before receiving SIGTERM signal from Kubernetes but no longer than pod's `terminationGracePeriodSeconds`. If not provided, it will be defaulted to `0`", + }, } } diff --git a/cli/cmd/inject.go b/cli/cmd/inject.go index b5716e509df3d..3ccd2cf4f0e07 100644 --- a/cli/cmd/inject.go +++ b/cli/cmd/inject.go @@ -100,6 +100,11 @@ sub-folders, or coming from stdin.`, &manualOption, "manual", manualOption, "Include the proxy sidecar container spec in the YAML output (the auto-injector won't pick it up, so config annotations aren't supported) (default false)", ) + flags.Uint64Var( + &options.waitBeforeExitSeconds, "wait-before-exit-seconds", options.waitBeforeExitSeconds, + "The period during which the proxy sidecar must stay alive while its pod is terminating. "+ + "Must be smaller than terminationGracePeriodSeconds for the pod (default 0)", + ) flags.BoolVar( &options.disableIdentity, "disable-identity", options.disableIdentity, "Disables resources from participating in TLS identity", @@ -439,6 +444,13 @@ func (options *proxyConfigOptions) overrideConfigs(configs *cfg.All, overrideAnn if options.traceCollectorSvcAccount != "" { overrideAnnotations[k8s.ProxyTraceCollectorSvcAccountAnnotation] = options.traceCollectorSvcAccount } + if options.waitBeforeExitSeconds != 0 { + overrideAnnotations[k8s.ProxyWaitBeforeExitSecondsAnnotation] = uintToString(options.waitBeforeExitSeconds) + } +} + +func uintToString(v uint64) string { + return strconv.FormatUint(v, 10) } func toPort(p uint) *cfg.Port { diff --git a/cli/cmd/install.go b/cli/cmd/install.go index 5b3024ffe047a..e08d3da02e48a 100644 --- a/cli/cmd/install.go +++ b/cli/cmd/install.go @@ -198,6 +198,7 @@ func newInstallOptionsWithDefaults() (*installOptions, error) { proxyCPULimit: defaults.Proxy.Resources.CPU.Limit, proxyMemoryLimit: defaults.Proxy.Resources.Memory.Limit, enableExternalProfiles: defaults.Proxy.EnableExternalProfiles, + waitBeforeExitSeconds: defaults.Proxy.WaitBeforeExitSeconds, }, identityOptions: &installIdentityOptions{ trustDomain: defaults.Identity.TrustDomain, diff --git a/cli/cmd/root.go b/cli/cmd/root.go index bcd69dd689df3..ae95203ee44cd 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -199,6 +199,7 @@ type proxyConfigOptions struct { enableExternalProfiles bool traceCollector string traceCollectorSvcAccount string + waitBeforeExitSeconds uint64 // ignoreCluster is not validated by validate(). ignoreCluster bool disableIdentity bool diff --git a/pkg/charts/linkerd2/values.go b/pkg/charts/linkerd2/values.go index af7a902ace793..b27f209c7c663 100644 --- a/pkg/charts/linkerd2/values.go +++ b/pkg/charts/linkerd2/values.go @@ -95,6 +95,7 @@ type ( Resources *Resources `json:"resources"` Trace *Trace `json:"trace"` UID int64 `json:"uid"` + WaitBeforeExitSeconds uint64 `json:"waitBeforeExitSeconds"` } // ProxyInit contains the fields to set the proxy-init container diff --git a/pkg/charts/linkerd2/values_test.go b/pkg/charts/linkerd2/values_test.go index 9011d9d05be3a..f3daf163400d5 100644 --- a/pkg/charts/linkerd2/values_test.go +++ b/pkg/charts/linkerd2/values_test.go @@ -98,7 +98,8 @@ func TestNewValues(t *testing.T) { CollectorSvcAddr: "", CollectorSvcAccount: "default", }, - UID: 2102, + UID: 2102, + WaitBeforeExitSeconds: 0, }, ProxyInit: &ProxyInit{ diff --git a/pkg/inject/inject.go b/pkg/inject/inject.go index ddd737ec1f078..d3e989b8a21d1 100644 --- a/pkg/inject/inject.go +++ b/pkg/inject/inject.go @@ -467,8 +467,9 @@ func (conf *ResourceConfig) injectPodSpec(values *patch) { Inbound: conf.proxyInboundPort(), Outbound: conf.proxyOutboundPort(), }, - UID: conf.proxyUID(), - Resources: conf.proxyResourceRequirements(), + UID: conf.proxyUID(), + Resources: conf.proxyResourceRequirements(), + WaitBeforeExitSeconds: conf.proxyWaitBeforeExitSeconds(), } if v := conf.pod.meta.Annotations[k8s.ProxyEnableDebugAnnotation]; v != "" { @@ -757,6 +758,20 @@ func (conf *ResourceConfig) tapDisabled() bool { return false } +func (conf *ResourceConfig) proxyWaitBeforeExitSeconds() uint64 { + if override := conf.getOverride(k8s.ProxyWaitBeforeExitSecondsAnnotation); override != "" { + waitBeforeExitSeconds, err := strconv.ParseUint(override, 10, 64) + if nil != err { + log.Warnf("unrecognized value used for the %s annotation, uint64 is expected: %s", + k8s.ProxyWaitBeforeExitSecondsAnnotation, override) + } + + return waitBeforeExitSeconds + } + + return 0 +} + func (conf *ResourceConfig) proxyResourceRequirements() *l5dcharts.Resources { var ( requestCPU k8sResource.Quantity diff --git a/pkg/inject/inject_test.go b/pkg/inject/inject_test.go index 5806d9ad5b96e..99f8046fdc9bf 100644 --- a/pkg/inject/inject_test.go +++ b/pkg/inject/inject_test.go @@ -15,23 +15,24 @@ import ( ) type expectedProxyConfigs struct { - identityContext *config.IdentityContext - image string - imagePullPolicy string - proxyVersion string - controlPort int32 - inboundPort int32 - adminPort int32 - outboundPort int32 - logLevel string - resourceRequirements *l5dcharts.Resources - proxyUID int64 - initImage string - initImagePullPolicy string - initVersion string - inboundSkipPorts string - outboundSkipPorts string - trace *l5dcharts.Trace + identityContext *config.IdentityContext + image string + imagePullPolicy string + proxyVersion string + controlPort int32 + inboundPort int32 + adminPort int32 + outboundPort int32 + proxyWaitBeforeExitSeconds uint64 + logLevel string + resourceRequirements *l5dcharts.Resources + proxyUID int64 + initImage string + initImagePullPolicy string + initVersion string + inboundSkipPorts string + outboundSkipPorts string + trace *l5dcharts.Trace } func TestConfigAccessors(t *testing.T) { @@ -107,20 +108,22 @@ func TestConfigAccessors(t *testing.T) { k8s.ProxyVersionOverrideAnnotation: proxyVersionOverride, k8s.ProxyTraceCollectorSvcAddrAnnotation: "oc-collector.tracing:55678", k8s.ProxyTraceCollectorSvcAccountAnnotation: "default", + k8s.ProxyWaitBeforeExitSecondsAnnotation: "123", }, }, Spec: corev1.PodSpec{}, }, }, expected: expectedProxyConfigs{ - image: "gcr.io/linkerd-io/proxy", - imagePullPolicy: "Always", - proxyVersion: proxyVersionOverride, - controlPort: int32(4000), - inboundPort: int32(5000), - adminPort: int32(5001), - outboundPort: int32(5002), - logLevel: "debug,linkerd2_proxy=debug", + image: "gcr.io/linkerd-io/proxy", + imagePullPolicy: "Always", + proxyVersion: proxyVersionOverride, + controlPort: int32(4000), + inboundPort: int32(5000), + adminPort: int32(5001), + outboundPort: int32(5002), + proxyWaitBeforeExitSeconds: 123, + logLevel: "debug,linkerd2_proxy=debug", resourceRequirements: &l5dcharts.Resources{ CPU: l5dcharts.Constraints{ Limit: "1500m", @@ -151,15 +154,16 @@ func TestConfigAccessors(t *testing.T) { }, }, expected: expectedProxyConfigs{ - identityContext: &config.IdentityContext{}, - image: "gcr.io/linkerd-io/proxy", - imagePullPolicy: "IfNotPresent", - proxyVersion: proxyVersion, - controlPort: int32(9000), - inboundPort: int32(6000), - adminPort: int32(6001), - outboundPort: int32(6002), - logLevel: "info,linkerd2_proxy=debug", + identityContext: &config.IdentityContext{}, + image: "gcr.io/linkerd-io/proxy", + imagePullPolicy: "IfNotPresent", + proxyVersion: proxyVersion, + controlPort: int32(9000), + inboundPort: int32(6000), + adminPort: int32(6001), + outboundPort: int32(6002), + proxyWaitBeforeExitSeconds: 0, + logLevel: "info,linkerd2_proxy=debug", resourceRequirements: &l5dcharts.Resources{ CPU: l5dcharts.Constraints{ Limit: "1", @@ -200,6 +204,7 @@ func TestConfigAccessors(t *testing.T) { k8s.ProxyVersionOverrideAnnotation: proxyVersionOverride, k8s.ProxyTraceCollectorSvcAddrAnnotation: "oc-collector.tracing:55678", k8s.ProxyTraceCollectorSvcAccountAnnotation: "default", + k8s.ProxyWaitBeforeExitSecondsAnnotation: "123", }, spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ @@ -207,14 +212,15 @@ func TestConfigAccessors(t *testing.T) { }, }, expected: expectedProxyConfigs{ - image: "gcr.io/linkerd-io/proxy", - imagePullPolicy: "Always", - proxyVersion: proxyVersionOverride, - controlPort: int32(4000), - inboundPort: int32(5000), - adminPort: int32(5001), - outboundPort: int32(5002), - logLevel: "debug,linkerd2_proxy=debug", + image: "gcr.io/linkerd-io/proxy", + imagePullPolicy: "Always", + proxyVersion: proxyVersionOverride, + controlPort: int32(4000), + inboundPort: int32(5000), + adminPort: int32(5001), + outboundPort: int32(5002), + proxyWaitBeforeExitSeconds: 123, + logLevel: "debug,linkerd2_proxy=debug", resourceRequirements: &l5dcharts.Resources{ CPU: l5dcharts.Constraints{ Limit: "1500m", @@ -237,6 +243,45 @@ func TestConfigAccessors(t *testing.T) { }, }, }, + {id: "use not a uint value for ProxyWaitBeforeExitSecondsAnnotation annotation", + nsAnnotations: map[string]string{ + k8s.ProxyWaitBeforeExitSecondsAnnotation: "-111", + }, + spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: corev1.PodSpec{}, + }, + }, + expected: expectedProxyConfigs{ + identityContext: &config.IdentityContext{}, + image: "gcr.io/linkerd-io/proxy", + imagePullPolicy: "IfNotPresent", + proxyVersion: proxyVersion, + controlPort: int32(9000), + inboundPort: int32(6000), + adminPort: int32(6001), + outboundPort: int32(6002), + proxyWaitBeforeExitSeconds: 0, + logLevel: "info,linkerd2_proxy=debug", + resourceRequirements: &l5dcharts.Resources{ + CPU: l5dcharts.Constraints{ + Limit: "1", + Request: "200m", + }, + Memory: l5dcharts.Constraints{ + Limit: "128", + Request: "64", + }, + }, + proxyUID: int64(8888), + initImage: "gcr.io/linkerd-io/proxy-init", + initImagePullPolicy: "IfNotPresent", + initVersion: version.ProxyInitVersion, + inboundSkipPorts: "53", + outboundSkipPorts: "9079", + }, + }, } for _, tc := range testCases { @@ -315,6 +360,13 @@ func TestConfigAccessors(t *testing.T) { } }) + t.Run("proxyWaitBeforeExitSeconds", func(t *testing.T) { + expected := testCase.expected.proxyWaitBeforeExitSeconds + if actual := resourceConfig.proxyWaitBeforeExitSeconds(); expected != actual { + t.Errorf("Expected: %v Actual: %v", expected, actual) + } + }) + t.Run("proxyLogLevel", func(t *testing.T) { expected := testCase.expected.logLevel if actual := resourceConfig.proxyLogLevel(); expected != actual { diff --git a/pkg/k8s/labels.go b/pkg/k8s/labels.go index 72029b2e1ecd9..bd3c559fae8a5 100644 --- a/pkg/k8s/labels.go +++ b/pkg/k8s/labels.go @@ -102,6 +102,9 @@ const ( // ProxyConfigAnnotationsPrefix is the prefix of all config-related annotations ProxyConfigAnnotationsPrefix = "config.linkerd.io" + // ProxyConfigAnnotationsPrefixAlpha is the prefix of newly released config-related annotations + ProxyConfigAnnotationsPrefixAlpha = "config.alpha.linkerd.io" + // ProxyImageAnnotation can be used to override the proxyImage config. ProxyImageAnnotation = ProxyConfigAnnotationsPrefix + "/proxy-image" @@ -178,10 +181,15 @@ const ( // its value. ProxyTraceCollectorSvcAddrAnnotation = ProxyConfigAnnotationsPrefix + "/trace-collector" + // ProxyWaitBeforeExitSecondsAnnotation makes the proxy container to wait for the given period before exiting + // after the Pod entered the Terminating state. Must be smaller than terminationGracePeriodSeconds + // configured for the Pod + ProxyWaitBeforeExitSecondsAnnotation = ProxyConfigAnnotationsPrefixAlpha + "/proxy-wait-before-exit-seconds" + // ProxyTraceCollectorSvcAccountAnnotation is used to specify the service account // associated with the trace collector. It is used to create the service's // mTLS identity. - ProxyTraceCollectorSvcAccountAnnotation = "config.alpha.linkerd.io/trace-collector-service-account" + ProxyTraceCollectorSvcAccountAnnotation = ProxyConfigAnnotationsPrefixAlpha + "/trace-collector-service-account" // IdentityModeDefault is assigned to IdentityModeAnnotation to // use the control plane's default identity scheme.