From 586052c0d06fea5e539885a2872d91de1a3a1ab7 Mon Sep 17 00:00:00 2001 From: Joao Marcal Date: Thu, 27 Apr 2023 19:12:39 +0100 Subject: [PATCH] operator: adds default PodAntiAffinity to ingester Signed-off-by: Joao Marcal --- operator/CHANGELOG.md | 1 + operator/internal/manifests/build_test.go | 25 +++-- operator/internal/manifests/compactor.go | 6 +- operator/internal/manifests/distributor.go | 7 +- operator/internal/manifests/gateway.go | 7 +- operator/internal/manifests/indexgateway.go | 7 +- operator/internal/manifests/ingester.go | 6 +- operator/internal/manifests/ingester_test.go | 38 ++++++++ operator/internal/manifests/querier.go | 7 +- operator/internal/manifests/query-frontend.go | 7 +- .../internal/manifests/query-frontend_test.go | 3 +- operator/internal/manifests/ruler.go | 7 +- operator/internal/manifests/ruler_test.go | 7 +- operator/internal/manifests/var.go | 94 +++++++++++++++---- 14 files changed, 158 insertions(+), 64 deletions(-) diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md index 2fe5830b62ac4..7d7e5925034af 100644 --- a/operator/CHANGELOG.md +++ b/operator/CHANGELOG.md @@ -1,4 +1,5 @@ ## Main +- [9329](https://github.com/grafana/loki/pull/9329) **JoaoBraveCoding**: Add default PodAntiAffinity to Ingester - [9262](https://github.com/grafana/loki/pull/9262) **btaani**: Add PodDisruptionBudget to the Ruler - [9260](https://github.com/grafana/loki/pull/9260) **JoaoBraveCoding**: Add PodDisruptionBudgets to the ingestion path - [9188](https://github.com/grafana/loki/pull/9188) **aminesnow**: Add PodDisruptionBudgets to the query path diff --git a/operator/internal/manifests/build_test.go b/operator/internal/manifests/build_test.go index bad81afc675ff..25eda861ff3d1 100644 --- a/operator/internal/manifests/build_test.go +++ b/operator/internal/manifests/build_test.go @@ -909,19 +909,19 @@ func TestBuildAll_WithFeatureGates_LokiStackAlerts(t *testing.T) { func TestBuildAll_WithFeatureGates_DefaultNodeAffinity(t *testing.T) { tt := []struct { - desc string - nodeAffinity bool - wantAffinity *corev1.Affinity + desc string + nodeAffinity bool + wantNodeAffinity *corev1.NodeAffinity }{ { - desc: "disabled", - nodeAffinity: false, - wantAffinity: nil, + desc: "disabled", + nodeAffinity: false, + wantNodeAffinity: nil, }, { - desc: "enabled", - nodeAffinity: true, - wantAffinity: defaultAffinity(true), + desc: "enabled", + nodeAffinity: true, + wantNodeAffinity: defaultNodeAffinity(true), }, } @@ -956,7 +956,12 @@ func TestBuildAll_WithFeatureGates_DefaultNodeAffinity(t *testing.T) { continue } - require.Equal(t, tc.wantAffinity, gotAffinity, + var gotNodeAffinity *corev1.NodeAffinity + if gotAffinity != nil { + gotNodeAffinity = gotAffinity.NodeAffinity + } + + require.Equal(t, tc.wantNodeAffinity, gotNodeAffinity, "kind", raw.GetObjectKind().GroupVersionKind(), "name", raw.GetName()) } diff --git a/operator/internal/manifests/compactor.go b/operator/internal/manifests/compactor.go index f619466b9bfe5..40cc31e8feb26 100644 --- a/operator/internal/manifests/compactor.go +++ b/operator/internal/manifests/compactor.go @@ -60,8 +60,10 @@ func BuildCompactor(opts Options) ([]client.Object, error) { // NewCompactorStatefulSet creates a statefulset object for a compactor. func NewCompactorStatefulSet(opts Options) *appsv1.StatefulSet { + l := ComponentLabels(LabelCompactorComponent, opts.Name) + a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) podSpec := corev1.PodSpec{ - Affinity: defaultAffinity(opts.Gates.DefaultNodeAffinity), + Affinity: configureAffinity(l, opts.Gates.DefaultNodeAffinity), Volumes: []corev1.Volume{ { Name: configVolumeName, @@ -129,8 +131,6 @@ func NewCompactorStatefulSet(opts Options) *appsv1.StatefulSet { podSpec.NodeSelector = opts.Stack.Template.Compactor.NodeSelector } - l := ComponentLabels(LabelCompactorComponent, opts.Name) - a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) return &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "StatefulSet", diff --git a/operator/internal/manifests/distributor.go b/operator/internal/manifests/distributor.go index 8f4eaf1f92d27..d099ad1314137 100644 --- a/operator/internal/manifests/distributor.go +++ b/operator/internal/manifests/distributor.go @@ -56,8 +56,10 @@ func BuildDistributor(opts Options) ([]client.Object, error) { // NewDistributorDeployment creates a deployment object for a distributor func NewDistributorDeployment(opts Options) *appsv1.Deployment { + l := ComponentLabels(LabelDistributorComponent, opts.Name) + a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) podSpec := corev1.PodSpec{ - Affinity: defaultAffinity(opts.Gates.DefaultNodeAffinity), + Affinity: configureAffinity(l, opts.Gates.DefaultNodeAffinity), Volumes: []corev1.Volume{ { Name: configVolumeName, @@ -125,9 +127,6 @@ func NewDistributorDeployment(opts Options) *appsv1.Deployment { podSpec.NodeSelector = opts.Stack.Template.Distributor.NodeSelector } - l := ComponentLabels(LabelDistributorComponent, opts.Name) - a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) - return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", diff --git a/operator/internal/manifests/gateway.go b/operator/internal/manifests/gateway.go index 17a05cca89a16..54bc42e812057 100644 --- a/operator/internal/manifests/gateway.go +++ b/operator/internal/manifests/gateway.go @@ -89,9 +89,11 @@ func BuildGateway(opts Options) ([]client.Object, error) { // NewGatewayDeployment creates a deployment object for a lokiStack-gateway func NewGatewayDeployment(opts Options, sha1C string) *appsv1.Deployment { + l := ComponentLabels(LabelGatewayComponent, opts.Name) + a := commonAnnotations(sha1C, opts.CertRotationRequiredAt) podSpec := corev1.PodSpec{ ServiceAccountName: GatewayName(opts.Name), - Affinity: defaultAffinity(opts.Gates.DefaultNodeAffinity), + Affinity: configureAffinity(l, opts.Gates.DefaultNodeAffinity), Volumes: []corev1.Volume{ { Name: "rbac", @@ -205,9 +207,6 @@ func NewGatewayDeployment(opts Options, sha1C string) *appsv1.Deployment { podSpec.NodeSelector = opts.Stack.Template.Gateway.NodeSelector } - l := ComponentLabels(LabelGatewayComponent, opts.Name) - a := commonAnnotations(sha1C, opts.CertRotationRequiredAt) - return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", diff --git a/operator/internal/manifests/indexgateway.go b/operator/internal/manifests/indexgateway.go index a75492197db95..b8c4b2ebe6841 100644 --- a/operator/internal/manifests/indexgateway.go +++ b/operator/internal/manifests/indexgateway.go @@ -62,8 +62,10 @@ func BuildIndexGateway(opts Options) ([]client.Object, error) { // NewIndexGatewayStatefulSet creates a statefulset object for an index-gateway func NewIndexGatewayStatefulSet(opts Options) *appsv1.StatefulSet { + l := ComponentLabels(LabelIndexGatewayComponent, opts.Name) + a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) podSpec := corev1.PodSpec{ - Affinity: defaultAffinity(opts.Gates.DefaultNodeAffinity), + Affinity: configureAffinity(l, opts.Gates.DefaultNodeAffinity), Volumes: []corev1.Volume{ { Name: configVolumeName, @@ -131,9 +133,6 @@ func NewIndexGatewayStatefulSet(opts Options) *appsv1.StatefulSet { podSpec.NodeSelector = opts.Stack.Template.IndexGateway.NodeSelector } - l := ComponentLabels(LabelIndexGatewayComponent, opts.Name) - a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) - return &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "StatefulSet", diff --git a/operator/internal/manifests/ingester.go b/operator/internal/manifests/ingester.go index cf862b62a63ee..673e913a74c31 100644 --- a/operator/internal/manifests/ingester.go +++ b/operator/internal/manifests/ingester.go @@ -62,8 +62,10 @@ func BuildIngester(opts Options) ([]client.Object, error) { // NewIngesterStatefulSet creates a deployment object for an ingester func NewIngesterStatefulSet(opts Options) *appsv1.StatefulSet { + l := ComponentLabels(LabelIngesterComponent, opts.Name) + a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) podSpec := corev1.PodSpec{ - Affinity: defaultAffinity(opts.Gates.DefaultNodeAffinity), + Affinity: configureAffinity(l, opts.Gates.DefaultNodeAffinity), Volumes: []corev1.Volume{ { Name: configVolumeName, @@ -141,8 +143,6 @@ func NewIngesterStatefulSet(opts Options) *appsv1.StatefulSet { podSpec.NodeSelector = opts.Stack.Template.Ingester.NodeSelector } - l := ComponentLabels(LabelIngesterComponent, opts.Name) - a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) return &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "StatefulSet", diff --git a/operator/internal/manifests/ingester_test.go b/operator/internal/manifests/ingester_test.go index 48c135770f27f..05dc8763a7b7f 100644 --- a/operator/internal/manifests/ingester_test.go +++ b/operator/internal/manifests/ingester_test.go @@ -7,7 +7,11 @@ import ( "github.com/stretchr/testify/require" policyv1 "k8s.io/api/policy/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + v1 "github.com/grafana/loki/operator/apis/config/v1" + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" "github.com/grafana/loki/operator/internal/manifests" "github.com/grafana/loki/operator/internal/manifests/internal" @@ -133,3 +137,37 @@ func TestBuildIngester_PodDisruptionBudget(t *testing.T) { }) } } + +func TestIngesterPodAntiAffinity(t *testing.T) { + sts := manifests.NewIngesterStatefulSet(manifests.Options{ + Name: "abcd", + Namespace: "efgh", + Stack: lokiv1.LokiStackSpec{ + StorageClassName: "standard", + Template: &lokiv1.LokiTemplateSpec{ + Ingester: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + }) + expectedPodAntiAffinity := &corev1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ + { + Weight: 100, + PodAffinityTerm: corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/component": manifests.LabelIngesterComponent, + "app.kubernetes.io/instance": "abcd", + + }, + }, + TopologyKey: "kubernetes.io/hostname", + }, + }, + }, + } + require.NotNil(t, sts.Spec.Template.Spec.Affinity) + require.Equal(t, expectedPodAntiAffinity, sts.Spec.Template.Spec.Affinity.PodAntiAffinity) +} diff --git a/operator/internal/manifests/querier.go b/operator/internal/manifests/querier.go index b0f8d20cb5fa4..ae8693e2e2e30 100644 --- a/operator/internal/manifests/querier.go +++ b/operator/internal/manifests/querier.go @@ -62,8 +62,10 @@ func BuildQuerier(opts Options) ([]client.Object, error) { // NewQuerierDeployment creates a deployment object for a querier func NewQuerierDeployment(opts Options) *appsv1.Deployment { + l := ComponentLabels(LabelQuerierComponent, opts.Name) + a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) podSpec := corev1.PodSpec{ - Affinity: defaultAffinity(opts.Gates.DefaultNodeAffinity), + Affinity: configureAffinity(l, opts.Gates.DefaultNodeAffinity), Volumes: []corev1.Volume{ { Name: configVolumeName, @@ -131,9 +133,6 @@ func NewQuerierDeployment(opts Options) *appsv1.Deployment { podSpec.NodeSelector = opts.Stack.Template.Querier.NodeSelector } - l := ComponentLabels(LabelQuerierComponent, opts.Name) - a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) - return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", diff --git a/operator/internal/manifests/query-frontend.go b/operator/internal/manifests/query-frontend.go index 9a67cba83066f..331775003037f 100644 --- a/operator/internal/manifests/query-frontend.go +++ b/operator/internal/manifests/query-frontend.go @@ -56,8 +56,10 @@ func BuildQueryFrontend(opts Options) ([]client.Object, error) { // NewQueryFrontendDeployment creates a deployment object for a query-frontend func NewQueryFrontendDeployment(opts Options) *appsv1.Deployment { + l := ComponentLabels(LabelQueryFrontendComponent, opts.Name) + a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) podSpec := corev1.PodSpec{ - Affinity: defaultAffinity(opts.Gates.DefaultNodeAffinity), + Affinity: configureAffinity(l, opts.Gates.DefaultNodeAffinity), Volumes: []corev1.Volume{ { Name: configVolumeName, @@ -137,9 +139,6 @@ func NewQueryFrontendDeployment(opts Options) *appsv1.Deployment { podSpec.NodeSelector = opts.Stack.Template.QueryFrontend.NodeSelector } - l := ComponentLabels(LabelQueryFrontendComponent, opts.Name) - a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) - return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ Kind: "Deployment", diff --git a/operator/internal/manifests/query-frontend_test.go b/operator/internal/manifests/query-frontend_test.go index 09d0fdb4d70ff..e078befb03d44 100644 --- a/operator/internal/manifests/query-frontend_test.go +++ b/operator/internal/manifests/query-frontend_test.go @@ -3,9 +3,10 @@ package manifests import ( "testing" - lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" "github.com/stretchr/testify/require" policyv1 "k8s.io/api/policy/v1" + + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" ) func TestNewQueryFrontendDeployment_SelectorMatchesLabels(t *testing.T) { diff --git a/operator/internal/manifests/ruler.go b/operator/internal/manifests/ruler.go index f0d75ce0ff225..5bde23d63db77 100644 --- a/operator/internal/manifests/ruler.go +++ b/operator/internal/manifests/ruler.go @@ -81,8 +81,10 @@ func NewRulerStatefulSet(opts Options) *appsv1.StatefulSet { }) } + l := ComponentLabels(LabelRulerComponent, opts.Name) + a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) podSpec := corev1.PodSpec{ - Affinity: defaultAffinity(opts.Gates.DefaultNodeAffinity), + Affinity: configureAffinity(l, opts.Gates.DefaultNodeAffinity), Volumes: []corev1.Volume{ { Name: configVolumeName, @@ -173,9 +175,6 @@ func NewRulerStatefulSet(opts Options) *appsv1.StatefulSet { podSpec.NodeSelector = opts.Stack.Template.Ruler.NodeSelector } - l := ComponentLabels(LabelRulerComponent, opts.Name) - a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) - return &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "StatefulSet", diff --git a/operator/internal/manifests/ruler_test.go b/operator/internal/manifests/ruler_test.go index ed17a06d73b94..6242d33ed6ed4 100644 --- a/operator/internal/manifests/ruler_test.go +++ b/operator/internal/manifests/ruler_test.go @@ -4,12 +4,13 @@ import ( "math/rand" "testing" - lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" - "github.com/grafana/loki/operator/internal/manifests" - "github.com/grafana/loki/operator/internal/manifests/openshift" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" + + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/internal/manifests" + "github.com/grafana/loki/operator/internal/manifests/openshift" ) func TestNewRulerStatefulSet_HasTemplateConfigHashAnnotation(t *testing.T) { diff --git a/operator/internal/manifests/var.go b/operator/internal/manifests/var.go index 41f8f2de9beb0..fce7cc6c97cc8 100644 --- a/operator/internal/manifests/var.go +++ b/operator/internal/manifests/var.go @@ -4,12 +4,14 @@ import ( "fmt" "path" - "github.com/grafana/loki/operator/internal/manifests/openshift" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" + + "github.com/grafana/loki/operator/internal/manifests/openshift" ) const ( @@ -99,13 +101,19 @@ const ( // caFile is the file name of the certificate authority file caFile = "service-ca.crt" - kubernetesNodeOSLabel = "kubernetes.io/os" - kubernetesNodeOSLinux = "linux" + kubernetesNodeOSLabel = "kubernetes.io/os" + kubernetesNodeOSLinux = "linux" + kubernetesNodeHostnameLabel = "kubernetes.io/hostname" + kubernetesCompomentLabel = "app.kubernetes.io/component" + kubernetesInstanceLabel = "app.kubernetes.io/instance" ) var ( - defaultConfigMapMode = int32(420) - volumeFileSystemMode = corev1.PersistentVolumeFilesystem + defaultConfigMapMode = int32(420) + volumeFileSystemMode = corev1.PersistentVolumeFilesystem + podAntiAffinityComponents = map[string]struct{}{ + LabelIngesterComponent: {}, + } ) func commonAnnotations(configHash, rotationRequiredAt string) map[string]string { @@ -118,7 +126,7 @@ func commonAnnotations(configHash, rotationRequiredAt string) map[string]string func commonLabels(stackName string) map[string]string { return map[string]string{ "app.kubernetes.io/name": "lokistack", - "app.kubernetes.io/instance": stackName, + kubernetesInstanceLabel: stackName, "app.kubernetes.io/managed-by": "lokistack-controller", "app.kubernetes.io/created-by": "lokistack-controller", } @@ -135,7 +143,7 @@ func serviceAnnotations(serviceName string, enableSigningService bool) map[strin // ComponentLabels is a list of all commonLabels including the app.kubernetes.io/component: label func ComponentLabels(component, stackName string) labels.Set { return labels.Merge(commonLabels(stackName), map[string]string{ - "app.kubernetes.io/component": component, + kubernetesCompomentLabel: component, }) } @@ -455,23 +463,37 @@ func gatewayServiceMonitorEndpoint(gatewayName, portName, serviceName, namespace } } -func defaultAffinity(enableNodeAffinity bool) *corev1.Affinity { +// configureAffinity returns an Affinity struture that can be used directly +// in a Deployment/StatefulSet. Parameters will affected configuration of the +// different fields in Affinity (NodeAffinity, PodAffinity, PodAntiAffinity). +func configureAffinity(labels labels.Set, enableNodeAffinity bool) *corev1.Affinity { + affinity := &corev1.Affinity{ + NodeAffinity: defaultNodeAffinity(enableNodeAffinity), + PodAntiAffinity: defaultPodAntiAffinity(labels), + } + + if affinity.NodeAffinity == nil && affinity.PodAntiAffinity == nil { + return nil + } + return affinity +} + +// defaultNodeAffinity if enabled will require pods to run on Linux nodes +func defaultNodeAffinity(enableNodeAffinity bool) *corev1.NodeAffinity { if !enableNodeAffinity { return nil } - return &corev1.Affinity{ - NodeAffinity: &corev1.NodeAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: kubernetesNodeOSLabel, - Operator: corev1.NodeSelectorOpIn, - Values: []string{ - kubernetesNodeOSLinux, - }, + return &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: kubernetesNodeOSLabel, + Operator: corev1.NodeSelectorOpIn, + Values: []string{ + kubernetesNodeOSLinux, }, }, }, @@ -481,6 +503,38 @@ func defaultAffinity(enableNodeAffinity bool) *corev1.Affinity { } } +// defaultPodAntiAffinity for components in podAntiAffinityComponents will +// configure pods, of a LokiStack, to preferably not run on the same node +func defaultPodAntiAffinity(labels labels.Set) *corev1.PodAntiAffinity { + // This code assumes that this function will never be called with a set of labels + // that don't have the "component" and "instance" labels since we enforce those on + // all the components of the LokiStack + componentLabel := labels[kubernetesCompomentLabel] + stackName := labels[kubernetesInstanceLabel] + + _, enablePodAntiAffinity := podAntiAffinityComponents[componentLabel] + if !enablePodAntiAffinity { + return nil + } + + return &corev1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{ + { + Weight: 100, + PodAffinityTerm: corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/component": componentLabel, + "app.kubernetes.io/instance": stackName, + }, + }, + TopologyKey: kubernetesNodeHostnameLabel, + }, + }, + }, + } +} + func lokiLivenessProbe() *corev1.Probe { return &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{