From abf08e2411c752caca81beb94bd0e65c4b9bac50 Mon Sep 17 00:00:00 2001 From: Periklis Tsirakidis Date: Thu, 3 Nov 2022 20:12:07 +0100 Subject: [PATCH] operator: Add support for built-in-cert-rotation for all internal lokistack encryption (#7064) --- .../apis/config/v1/projectconfig_types.go | 34 +- .../apis/config/v1/zz_generated.deepcopy.go | 16 + operator/apis/loki/v1/lokistack_types.go | 2 + ...-operator-manager-config_v1_configmap.yaml | 10 + .../loki-operator.clusterserviceversion.yaml | 1 + operator/cmd/loki-broker/main.go | 9 +- .../manager_related_image_patch.yaml | 2 +- .../openshift/controller_manager_config.yaml | 10 + operator/config/rbac/role.yaml | 1 + .../loki/certrotation_controller.go | 113 +++ .../loki/certrotation_controller_test.go | 74 ++ .../lokistack/certrotation_discovery.go | 45 + .../controllers/loki/lokistack_controller.go | 35 +- .../loki/lokistack_controller_test.go | 52 +- operator/go.mod | 2 +- operator/hack/addons_kind_certs.yaml | 40 +- operator/internal/certrotation/build.go | 76 ++ operator/internal/certrotation/build_test.go | 134 +++ operator/internal/certrotation/cabundle.go | 86 ++ .../internal/certrotation/cabundle_test.go | 74 ++ operator/internal/certrotation/expiry.go | 17 + operator/internal/certrotation/options.go | 86 ++ operator/internal/certrotation/rotation.go | 184 ++++ .../internal/certrotation/rotation_test.go | 335 +++++++ operator/internal/certrotation/signer.go | 100 ++ operator/internal/certrotation/signer_test.go | 131 +++ operator/internal/certrotation/target.go | 126 +++ operator/internal/certrotation/target_test.go | 165 ++++ operator/internal/certrotation/var.go | 52 + .../handlers/internal/certificates/options.go | 128 +++ .../internal/certificates/options_test.go | 217 +++++ .../serviceaccounts/serviceaccounts.go | 22 + .../handlers/lokistack_check_cert_expiry.go | 58 ++ .../lokistack_check_cert_expiry_test.go | 187 ++++ .../handlers/lokistack_create_or_update.go | 61 +- .../lokistack_create_or_update_test.go | 8 +- .../handlers/lokistack_rotate_certs.go | 110 +++ .../handlers/lokistack_rotate_certs_test.go | 566 +++++++++++ operator/internal/manifests/build.go | 4 - operator/internal/manifests/build_test.go | 32 +- operator/internal/manifests/compactor.go | 21 +- operator/internal/manifests/compactor_test.go | 20 + operator/internal/manifests/distributor.go | 50 +- .../internal/manifests/distributor_test.go | 22 +- operator/internal/manifests/gateway.go | 211 +++- .../internal/manifests/gateway_tenants.go | 63 +- .../manifests/gateway_tenants_test.go | 905 ++++-------------- operator/internal/manifests/gateway_test.go | 223 ++++- operator/internal/manifests/indexgateway.go | 21 +- .../internal/manifests/indexgateway_test.go | 20 + operator/internal/manifests/ingester.go | 53 +- operator/internal/manifests/ingester_test.go | 20 + operator/internal/manifests/mutate.go | 24 +- operator/internal/manifests/mutate_test.go | 42 +- .../internal/manifests/openshift/build.go | 11 +- .../manifests/openshift/build_test.go | 48 +- .../internal/manifests/openshift/configure.go | 196 +--- .../manifests/openshift/opa_openshift.go | 6 +- .../manifests/openshift/service_ca.go | 24 +- .../manifests/openshift/serviceaccount.go | 26 +- .../openshift/serviceaccount_test.go | 44 - operator/internal/manifests/openshift/var.go | 13 +- operator/internal/manifests/options.go | 13 +- operator/internal/manifests/querier.go | 54 +- operator/internal/manifests/querier_test.go | 20 + operator/internal/manifests/query-frontend.go | 97 +- .../internal/manifests/query-frontend_test.go | 160 +--- operator/internal/manifests/ruler.go | 153 ++- operator/internal/manifests/ruler_test.go | 51 + operator/internal/manifests/service.go | 76 +- .../internal/manifests/service_monitor.go | 21 +- .../manifests/service_monitor_test.go | 409 +++++++- operator/internal/manifests/service_test.go | 711 ++++++++++++++ operator/internal/manifests/var.go | 209 +++- operator/main.go | 31 +- operator/quickstart.sh | 5 +- 76 files changed, 5754 insertions(+), 1724 deletions(-) create mode 100644 operator/controllers/loki/certrotation_controller.go create mode 100644 operator/controllers/loki/certrotation_controller_test.go create mode 100644 operator/controllers/loki/internal/lokistack/certrotation_discovery.go create mode 100644 operator/internal/certrotation/build.go create mode 100644 operator/internal/certrotation/build_test.go create mode 100644 operator/internal/certrotation/cabundle.go create mode 100644 operator/internal/certrotation/cabundle_test.go create mode 100644 operator/internal/certrotation/expiry.go create mode 100644 operator/internal/certrotation/options.go create mode 100644 operator/internal/certrotation/rotation.go create mode 100644 operator/internal/certrotation/rotation_test.go create mode 100644 operator/internal/certrotation/signer.go create mode 100644 operator/internal/certrotation/signer_test.go create mode 100644 operator/internal/certrotation/target.go create mode 100644 operator/internal/certrotation/target_test.go create mode 100644 operator/internal/certrotation/var.go create mode 100644 operator/internal/handlers/internal/certificates/options.go create mode 100644 operator/internal/handlers/internal/certificates/options_test.go create mode 100644 operator/internal/handlers/internal/serviceaccounts/serviceaccounts.go create mode 100644 operator/internal/handlers/lokistack_check_cert_expiry.go create mode 100644 operator/internal/handlers/lokistack_check_cert_expiry_test.go create mode 100644 operator/internal/handlers/lokistack_rotate_certs.go create mode 100644 operator/internal/handlers/lokistack_rotate_certs_test.go delete mode 100644 operator/internal/manifests/openshift/serviceaccount_test.go diff --git a/operator/apis/config/v1/projectconfig_types.go b/operator/apis/config/v1/projectconfig_types.go index 8b553b6f20bb..cc13f80fc448 100644 --- a/operator/apis/config/v1/projectconfig_types.go +++ b/operator/apis/config/v1/projectconfig_types.go @@ -5,9 +5,30 @@ import ( cfg "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" ) +// BuiltInCertManagement is the configuration for the built-in facility to generate and rotate +// TLS client and serving certificates for all LokiStack services and internal clients except +// for the lokistack-gateway. +type BuiltInCertManagement struct { + // Enabled defines to flag to enable/disable built-in certificate management feature gate. + Enabled bool `json:"enabled,omitempty"` + // CACertValidity defines the total duration of the CA certificate validity. + CACertValidity string `json:"caValidity,omitempty"` + // CACertRefresh defines the duration of the CA certificate validity until a rotation + // should happen. It can be set up to 80% of CA certificate validity or equal to the + // CA certificate validity. Latter should be used only for rotating only when expired. + CACertRefresh string `json:"caRefresh,omitempty"` + // CertValidity defines the total duration of the validity for all LokiStack certificates. + CertValidity string `json:"certValidity,omitempty"` + // CertRefresh defines the duration of the certificate validity until a rotation + // should happen. It can be set up to 80% of certificate validity or equal to the + // certificate validity. Latter should be used only for rotating only when expired. + // The refresh is applied to all LokiStack certificates at once. + CertRefresh string `json:"certRefresh,omitempty"` +} + // OpenShiftFeatureGates is the supported set of all operator features gates on OpenShift. type OpenShiftFeatureGates struct { - // ServingCertsService enables OpenShift service-ca annotations on Services + // ServingCertsService enables OpenShift service-ca annotations on the lokistack-gateway service only // to use the in-platform CA and generate a TLS cert/key pair per service for // in-cluster data-in-transit encryption. // More details: https://docs.openshift.com/container-platform/latest/security/certificate_types_descriptions/service-ca-certificates.html @@ -54,6 +75,17 @@ type FeatureGates struct { // suffix `-ca-bundle`, e.g. `lokistack-dev-ca-bundle` and the following data: // - `service-ca.crt`: The CA signing the service certificate in `tls.crt`. GRPCEncryption bool `json:"grpcEncryption,omitempty"` + // BuiltInCertManagement enables the built-in facility for generating and rotating + // TLS client and serving certificates for all LokiStack services and internal clients except + // for the lokistack-gateway, In detail all internal Loki HTTP and GRPC communication is lifted + // to require mTLS. For the lokistack-gateay you need to provide a secret with or use the `ServingCertsService` + // on OpenShift: + // - `tls.crt`: The TLS server side certificate. + // - `tls.key`: The TLS key for server-side encryption. + // In addition each service requires a configmap named as the LokiStack CR with the + // suffix `-ca-bundle`, e.g. `lokistack-dev-ca-bundle` and the following data: + // - `service-ca.crt`: The CA signing the service certificate in `tls.crt`. + BuiltInCertManagement BuiltInCertManagement `json:"builtInCertManagement,omitempty"` // LokiStackGateway enables reconciling the reverse-proxy lokistack-gateway // component for multi-tenant authentication/authorization traffic control diff --git a/operator/apis/config/v1/zz_generated.deepcopy.go b/operator/apis/config/v1/zz_generated.deepcopy.go index 3ff10850cd1a..c85446c21e0b 100644 --- a/operator/apis/config/v1/zz_generated.deepcopy.go +++ b/operator/apis/config/v1/zz_generated.deepcopy.go @@ -9,9 +9,25 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BuiltInCertManagement) DeepCopyInto(out *BuiltInCertManagement) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BuiltInCertManagement. +func (in *BuiltInCertManagement) DeepCopy() *BuiltInCertManagement { + if in == nil { + return nil + } + out := new(BuiltInCertManagement) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FeatureGates) DeepCopyInto(out *FeatureGates) { *out = *in + out.BuiltInCertManagement = in.BuiltInCertManagement out.OpenShift = in.OpenShift } diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go index da24185c4c0a..901331dc8eb5 100644 --- a/operator/apis/loki/v1/lokistack_types.go +++ b/operator/apis/loki/v1/lokistack_types.go @@ -761,6 +761,8 @@ const ( ReasonInvalidTenantsConfiguration LokiStackConditionReason = "InvalidTenantsConfiguration" // ReasonMissingGatewayOpenShiftBaseDomain when the reconciler cannot lookup the OpenShift DNS base domain. ReasonMissingGatewayOpenShiftBaseDomain LokiStackConditionReason = "MissingGatewayOpenShiftBaseDomain" + // ReasonFailedCertificateRotation when the reconciler cannot rotate any of the required TLS certificates. + ReasonFailedCertificateRotation LokiStackConditionReason = "FailedCertificateRotation" ) // PodStatusMap defines the type for mapping pod status to pod name. diff --git a/operator/bundle/manifests/loki-operator-manager-config_v1_configmap.yaml b/operator/bundle/manifests/loki-operator-manager-config_v1_configmap.yaml index 7b03313b8e93..a5f6abe18bf5 100644 --- a/operator/bundle/manifests/loki-operator-manager-config_v1_configmap.yaml +++ b/operator/bundle/manifests/loki-operator-manager-config_v1_configmap.yaml @@ -26,6 +26,16 @@ data: # httpEncryption: true grpcEncryption: true + builtInCertManagement: + enabled: true + # CA certificate validity: 5 years + caValidity: 43830h + # CA certificate refresh at 80% of validity + caRefresh: 35064h + # Target certificate validity: 90d + certValidity: 2160h + # Target certificate refresh at 80% of validity + certRefresh: 1728h # # Component feature gates # diff --git a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml index ba635e196b66..139861adb4c5 100644 --- a/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/manifests/loki-operator.clusterserviceversion.yaml @@ -984,6 +984,7 @@ spec: - endpoints - nodes - pods + - secrets - serviceaccounts - services verbs: diff --git a/operator/cmd/loki-broker/main.go b/operator/cmd/loki-broker/main.go index 0677b160d852..091221ce6655 100644 --- a/operator/cmd/loki-broker/main.go +++ b/operator/cmd/loki-broker/main.go @@ -37,10 +37,13 @@ func (c *config) registerFlags(f *flag.FlagSet) { f.StringVar(&c.Namespace, "namespace", "", "Namespace to deploy to") f.StringVar(&c.Image, "image", manifests.DefaultContainerImage, "The Loki image pull spec loation.") // Feature flags - c.featureFlags = configv1.FeatureGates{} - c.featureFlags.OpenShift = configv1.OpenShiftFeatureGates{} - f.BoolVar(&c.featureFlags.OpenShift.ServingCertsService, "with-serving-certs-service", false, "Enable usage of serving certs service on OpenShift.") f.BoolVar(&c.featureFlags.ServiceMonitors, "with-service-monitors", false, "Enable service monitors for all LokiStack components.") + f.BoolVar(&c.featureFlags.OpenShift.ServingCertsService, "with-serving-certs-service", false, "Enable usage of serving certs service on OpenShift.") + f.BoolVar(&c.featureFlags.BuiltInCertManagement.Enabled, "with-builtin-cert-management", false, "Enable usage built-in cert generation and rotation.") + f.StringVar(&c.featureFlags.BuiltInCertManagement.CACertValidity, "ca-cert-validity", "8760h", "CA Certificate validity duration.") + f.StringVar(&c.featureFlags.BuiltInCertManagement.CACertRefresh, "ca-cert-refresh", "7008h", "CA Certificate refresh time.") + f.StringVar(&c.featureFlags.BuiltInCertManagement.CertValidity, "target-cert-validity", "2160h", "Target Certificate validity duration.") + f.StringVar(&c.featureFlags.BuiltInCertManagement.CertRefresh, "target-cert-refresh", "1728h", "Target Certificate refresh time.") f.BoolVar(&c.featureFlags.HTTPEncryption, "with-http-tls-services", false, "Enables TLS for all LokiStack GRPC services.") f.BoolVar(&c.featureFlags.GRPCEncryption, "with-grpc-tls-services", false, "Enables TLS for all LokiStack HTTP services.") f.BoolVar(&c.featureFlags.ServiceMonitorTLSEndpoints, "with-service-monitor-tls-endpoints", false, "Enable TLS endpoint for service monitors.") diff --git a/operator/config/overlays/development/manager_related_image_patch.yaml b/operator/config/overlays/development/manager_related_image_patch.yaml index f26143a21cc1..e4c6fd9db5eb 100644 --- a/operator/config/overlays/development/manager_related_image_patch.yaml +++ b/operator/config/overlays/development/manager_related_image_patch.yaml @@ -9,6 +9,6 @@ spec: - name: manager env: - name: RELATED_IMAGE_LOKI - value: docker.io/grafana/loki:main-ec0bf70 + value: docker.io/grafana/loki:k120-26d2989 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest diff --git a/operator/config/overlays/openshift/controller_manager_config.yaml b/operator/config/overlays/openshift/controller_manager_config.yaml index c69ada2e8eda..d1e0d52a4242 100644 --- a/operator/config/overlays/openshift/controller_manager_config.yaml +++ b/operator/config/overlays/openshift/controller_manager_config.yaml @@ -23,6 +23,16 @@ featureGates: # httpEncryption: true grpcEncryption: true + builtInCertManagement: + enabled: true + # CA certificate validity: 5 years + caValidity: 43830h + # CA certificate refresh at 80% of validity + caRefresh: 35064h + # Target certificate validity: 90d + certValidity: 2160h + # Target certificate refresh at 80% of validity + certRefresh: 1728h # # Component feature gates # diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index 38c74b296090..806d79f83850 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -12,6 +12,7 @@ rules: - endpoints - nodes - pods + - secrets - serviceaccounts - services verbs: diff --git a/operator/controllers/loki/certrotation_controller.go b/operator/controllers/loki/certrotation_controller.go new file mode 100644 index 000000000000..382498cc4088 --- /dev/null +++ b/operator/controllers/loki/certrotation_controller.go @@ -0,0 +1,113 @@ +package controllers + +import ( + "context" + "errors" + "time" + + "github.com/go-logr/logr" + configv1 "github.com/grafana/loki/operator/apis/config/v1" + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/controllers/loki/internal/lokistack" + "github.com/grafana/loki/operator/controllers/loki/internal/management/state" + "github.com/grafana/loki/operator/internal/certrotation" + "github.com/grafana/loki/operator/internal/external/k8s" + "github.com/grafana/loki/operator/internal/handlers" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// CertRotationReconciler reconciles the `loki.grafana.com/certRotationRequiredAt` annotation on +// any LokiStack object associated with any of the owned signer/client/serving certificates secrets +// and CA bundle configmap. +type CertRotationReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + FeatureGates configv1.FeatureGates +} + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// Compare the state specified by the LokiStack object against the actual cluster state, +// and then perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.0/pkg/reconcile +func (r *CertRotationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + managed, err := state.IsManaged(ctx, req, r.Client) + if err != nil { + return ctrl.Result{ + Requeue: true, + }, err + } + if !managed { + r.Log.Info("Skipping reconciliation for unmanaged LokiStack resource", "name", req.String()) + // Stop requeueing for unmanaged LokiStack custom resources + return ctrl.Result{}, nil + } + + rt, err := certrotation.ParseRotation(r.FeatureGates.BuiltInCertManagement) + if err != nil { + return ctrl.Result{Requeue: false}, err + } + + checkExpiryAfter := expiryRetryAfter(rt.TargetCertRefresh) + r.Log.Info("Checking if LokiStack certificates expired", "name", req.String(), "interval", checkExpiryAfter.String()) + + var expired *certrotation.CertExpiredError + + err = handlers.CheckCertExpiry(ctx, r.Log, req, r.Client, r.FeatureGates) + switch { + case errors.As(err, &expired): + r.Log.Info("Certificate expired", "msg", expired.Error()) + case err != nil: + return ctrl.Result{ + Requeue: true, + }, err + default: + r.Log.Info("Skipping cert rotation, all LokiStack certificates still valid", "name", req.String()) + return ctrl.Result{ + RequeueAfter: checkExpiryAfter, + }, nil + } + + r.Log.Error(err, "LokiStack certificates expired", "name", req.String()) + err = lokistack.AnnotateForRequiredCertRotation(ctx, r.Client, req.Name, req.Namespace) + if err != nil { + r.Log.Error(err, "failed to annotate required cert rotation", "name", req.String()) + return ctrl.Result{ + Requeue: true, + }, err + } + + return ctrl.Result{ + RequeueAfter: checkExpiryAfter, + }, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *CertRotationReconciler) SetupWithManager(mgr ctrl.Manager) error { + b := ctrl.NewControllerManagedBy(mgr) + return r.buildController(k8s.NewCtrlBuilder(b)) +} + +func (r *CertRotationReconciler) buildController(bld k8s.Builder) error { + return bld. + For(&lokiv1.LokiStack{}). + Owns(&corev1.Secret{}). + Complete(r) +} + +func expiryRetryAfter(certRefresh time.Duration) time.Duration { + day := 24 * time.Hour + if certRefresh > day { + return 12 * time.Hour + } + + return certRefresh / 4 +} diff --git a/operator/controllers/loki/certrotation_controller_test.go b/operator/controllers/loki/certrotation_controller_test.go new file mode 100644 index 000000000000..4f3dd0ae2890 --- /dev/null +++ b/operator/controllers/loki/certrotation_controller_test.go @@ -0,0 +1,74 @@ +package controllers + +import ( + "testing" + "time" + + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/internal/external/k8s/k8sfakes" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" +) + +func TestCertRotationController_RegistersCustomResource_WithDefaultPredicates(t *testing.T) { + b := &k8sfakes.FakeBuilder{} + k := &k8sfakes.FakeClient{} + c := &CertRotationReconciler{Client: k, Scheme: scheme} + + b.ForReturns(b) + b.OwnsReturns(b) + + err := c.buildController(b) + require.NoError(t, err) + + // Require only one For-Call for the custom resource + require.Equal(t, 1, b.ForCallCount()) + + // Require For-call with LokiStack resource + obj, _ := b.ForArgsForCall(0) + require.Equal(t, &lokiv1.LokiStack{}, obj) +} + +func TestCertRotationController_RegisterOwnedResources_WithDefaultPredicates(t *testing.T) { + b := &k8sfakes.FakeBuilder{} + k := &k8sfakes.FakeClient{} + c := &CertRotationReconciler{Client: k, Scheme: scheme} + + b.ForReturns(b) + b.OwnsReturns(b) + + err := c.buildController(b) + require.NoError(t, err) + + require.Equal(t, 1, b.OwnsCallCount()) + + obj, _ := b.OwnsArgsForCall(0) + require.Equal(t, &corev1.Secret{}, obj) +} + +func TestCertRotationController_ExpiryRetryAfter(t *testing.T) { + tt := []struct { + desc string + refresh time.Duration + wantDuration time.Duration + wantError bool + }{ + { + desc: "multi-day refresh durarion", + refresh: 120 * time.Hour, + wantDuration: 12 * time.Hour, + }, + { + desc: "less than a day refresh duration", + refresh: 10 * time.Hour, + wantDuration: 2*time.Hour + 30*time.Minute, + }, + } + for _, tc := range tt { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + require.Equal(t, tc.wantDuration, expiryRetryAfter(tc.refresh)) + }) + } +} diff --git a/operator/controllers/loki/internal/lokistack/certrotation_discovery.go b/operator/controllers/loki/internal/lokistack/certrotation_discovery.go new file mode 100644 index 000000000000..c92e0115016c --- /dev/null +++ b/operator/controllers/loki/internal/lokistack/certrotation_discovery.go @@ -0,0 +1,45 @@ +package lokistack + +import ( + "context" + "fmt" + "time" + + "github.com/ViaQ/logerr/v2/kverrors" + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/internal/external/k8s" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const certRotationRequiredAtKey = "loki.grafana.com/certRotationRequiredAt" + +// AnnotateForRequiredCertRotation adds/updates the `loki.grafana.com/certRotationRequiredAt` annotation +// to the named Lokistack if any of the managed client/serving/ca certificates expired. If no LokiStack +// is found, then skip reconciliation. +func AnnotateForRequiredCertRotation(ctx context.Context, k k8s.Client, name, namespace string) error { + var s lokiv1.LokiStack + key := client.ObjectKey{Name: name, Namespace: namespace} + + if err := k.Get(ctx, key, &s); err != nil { + if apierrors.IsNotFound(err) { + // Do nothing + return nil + } + + return kverrors.Wrap(err, "failed to get lokistack", "key", key) + } + + ss := s.DeepCopy() + if ss.Annotations == nil { + ss.Annotations = make(map[string]string) + } + + ss.Annotations[certRotationRequiredAtKey] = time.Now().UTC().Format(time.RFC3339) + + if err := k.Update(ctx, ss); err != nil { + return kverrors.Wrap(err, fmt.Sprintf("failed to update lokistack `%s` annotation", certRotationRequiredAtKey), "key", key) + } + + return nil +} diff --git a/operator/controllers/loki/lokistack_controller.go b/operator/controllers/loki/lokistack_controller.go index d1afaa31a790..ecc98c2acc8d 100644 --- a/operator/controllers/loki/lokistack_controller.go +++ b/operator/controllers/loki/lokistack_controller.go @@ -77,7 +77,7 @@ type LokiStackReconciler struct { // +kubebuilder:rbac:groups=loki.grafana.com,resources=lokistacks,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=loki.grafana.com,resources=lokistacks/status,verbs=get;update;patch // +kubebuilder:rbac:groups=loki.grafana.com,resources=lokistacks/finalizers,verbs=update -// +kubebuilder:rbac:groups="",resources=pods;nodes;services;endpoints;configmaps;serviceaccounts,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=pods;nodes;services;endpoints;configmaps;secrets;serviceaccounts,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch // +kubebuilder:rbac:groups=apps,resources=deployments;statefulsets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings;clusterroles;roles;rolebindings,verbs=get;list;watch;create;update;patch;delete @@ -110,11 +110,33 @@ func (r *LokiStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, nil } + if r.FeatureGates.BuiltInCertManagement.Enabled { + err = handlers.CreateOrRotateCertificates(ctx, r.Log, req, r.Client, r.Scheme, r.FeatureGates) + if res, derr := handleDegradedError(ctx, r.Client, req, err); derr != nil { + return res, derr + } + } + err = handlers.CreateOrUpdateLokiStack(ctx, r.Log, req, r.Client, r.Scheme, r.FeatureGates) + if res, derr := handleDegradedError(ctx, r.Client, req, err); derr != nil { + return res, derr + } + + err = status.Refresh(ctx, r.Client, req) + if err != nil { + return ctrl.Result{ + Requeue: true, + RequeueAfter: time.Second, + }, err + } + return ctrl.Result{}, nil +} + +func handleDegradedError(ctx context.Context, c client.Client, req ctrl.Request, err error) (ctrl.Result, error) { var degraded *status.DegradedError if errors.As(err, °raded) { - err = status.SetDegradedCondition(ctx, r.Client, req, degraded.Message, degraded.Reason) + err = status.SetDegradedCondition(ctx, c, req, degraded.Message, degraded.Reason) if err != nil { return ctrl.Result{ Requeue: true, @@ -135,14 +157,6 @@ func (r *LokiStackReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( }, err } - err = status.Refresh(ctx, r.Client, req) - if err != nil { - return ctrl.Result{ - Requeue: true, - RequeueAfter: time.Second, - }, err - } - return ctrl.Result{}, nil } @@ -156,6 +170,7 @@ func (r *LokiStackReconciler) buildController(bld k8s.Builder) error { bld = bld. For(&lokiv1.LokiStack{}, createOrUpdateOnlyPred). Owns(&corev1.ConfigMap{}, updateOrDeleteOnlyPred). + Owns(&corev1.Secret{}, updateOrDeleteOnlyPred). Owns(&corev1.ServiceAccount{}, updateOrDeleteOnlyPred). Owns(&corev1.Service{}, updateOrDeleteOnlyPred). Owns(&appsv1.Deployment{}, updateOrDeleteOnlyPred). diff --git a/operator/controllers/loki/lokistack_controller_test.go b/operator/controllers/loki/lokistack_controller_test.go index 91f2c8bb18ee..0ff57aa4e1e5 100644 --- a/operator/controllers/loki/lokistack_controller_test.go +++ b/operator/controllers/loki/lokistack_controller_test.go @@ -85,55 +85,61 @@ func TestLokiStackController_RegisterOwnedResourcesForUpdateOrDeleteOnly(t *test { obj: &corev1.ConfigMap{}, index: 0, - ownCallsCount: 10, + ownCallsCount: 11, pred: updateOrDeleteOnlyPred, }, { - obj: &corev1.ServiceAccount{}, + obj: &corev1.Secret{}, index: 1, - ownCallsCount: 10, + ownCallsCount: 11, pred: updateOrDeleteOnlyPred, }, { - obj: &corev1.Service{}, + obj: &corev1.ServiceAccount{}, index: 2, - ownCallsCount: 10, + ownCallsCount: 11, pred: updateOrDeleteOnlyPred, }, { - obj: &appsv1.Deployment{}, + obj: &corev1.Service{}, index: 3, - ownCallsCount: 10, + ownCallsCount: 11, pred: updateOrDeleteOnlyPred, }, { - obj: &appsv1.StatefulSet{}, + obj: &appsv1.Deployment{}, index: 4, - ownCallsCount: 10, + ownCallsCount: 11, pred: updateOrDeleteOnlyPred, }, { - obj: &rbacv1.ClusterRole{}, + obj: &appsv1.StatefulSet{}, index: 5, - ownCallsCount: 10, + ownCallsCount: 11, pred: updateOrDeleteOnlyPred, }, { - obj: &rbacv1.ClusterRoleBinding{}, + obj: &rbacv1.ClusterRole{}, index: 6, - ownCallsCount: 10, + ownCallsCount: 11, pred: updateOrDeleteOnlyPred, }, { - obj: &rbacv1.Role{}, + obj: &rbacv1.ClusterRoleBinding{}, index: 7, - ownCallsCount: 10, + ownCallsCount: 11, pred: updateOrDeleteOnlyPred, }, { - obj: &rbacv1.RoleBinding{}, + obj: &rbacv1.Role{}, index: 8, - ownCallsCount: 10, + ownCallsCount: 11, + pred: updateOrDeleteOnlyPred, + }, + { + obj: &rbacv1.RoleBinding{}, + index: 9, + ownCallsCount: 11, pred: updateOrDeleteOnlyPred, }, // The next two share the same index, because the @@ -141,8 +147,8 @@ func TestLokiStackController_RegisterOwnedResourcesForUpdateOrDeleteOnly(t *test // or a Route (i.e. OpenShift). { obj: &networkingv1.Ingress{}, - index: 9, - ownCallsCount: 10, + index: 10, + ownCallsCount: 11, featureGates: configv1.FeatureGates{ OpenShift: configv1.OpenShiftFeatureGates{ GatewayRoute: false, @@ -152,8 +158,8 @@ func TestLokiStackController_RegisterOwnedResourcesForUpdateOrDeleteOnly(t *test }, { obj: &routev1.Route{}, - index: 9, - ownCallsCount: 10, + index: 10, + ownCallsCount: 11, featureGates: configv1.FeatureGates{ OpenShift: configv1.OpenShiftFeatureGates{ GatewayRoute: true, @@ -163,8 +169,8 @@ func TestLokiStackController_RegisterOwnedResourcesForUpdateOrDeleteOnly(t *test }, { obj: &openshiftconfigv1.APIServer{}, - index: 10, - ownCallsCount: 11, + index: 11, + ownCallsCount: 12, featureGates: configv1.FeatureGates{ OpenShift: configv1.OpenShiftFeatureGates{ ClusterTLSPolicy: true, diff --git a/operator/go.mod b/operator/go.mod index 20644e2f0e61..d9250c1988c4 100644 --- a/operator/go.mod +++ b/operator/go.mod @@ -28,6 +28,7 @@ require ( github.com/openshift/library-go v0.0.0-20220622115547-84d884f4c9f6 github.com/prometheus/prometheus v1.8.2-0.20220303173753-edfe657b5405 gopkg.in/yaml.v2 v2.4.0 + k8s.io/apiserver v0.25.0 ) require ( @@ -165,7 +166,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect inet.af/netaddr v0.0.0-20211027220019-c74959edd3b6 // indirect k8s.io/apiextensions-apiserver v0.25.0 // indirect - k8s.io/apiserver v0.25.0 // indirect k8s.io/component-base v0.25.0 // indirect k8s.io/klog/v2 v2.70.1 // indirect k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect diff --git a/operator/hack/addons_kind_certs.yaml b/operator/hack/addons_kind_certs.yaml index 45994b3f524e..5b5d4f23e983 100644 --- a/operator/hack/addons_kind_certs.yaml +++ b/operator/hack/addons_kind_certs.yaml @@ -8,14 +8,14 @@ spec: apiVersion: cert-manager.io/v1 kind: Certificate metadata: - name: lokistack-dev-ca-bundle + name: lokistack-dev-signing-ca spec: isCA: true commonName: lokistack-dev-gateway-http.default.svc dnsNames: - "*.default.svc" - "*.default.svc.cluster.local" - secretName: lokistack-dev-ca-bundle + secretName: lokistack-dev-signing-ca privateKey: algorithm: ECDSA size: 256 @@ -30,7 +30,7 @@ metadata: name: kind-issuer spec: ca: - secretName: lokistack-dev-ca-bundle + secretName: lokistack-dev-signing-ca --- apiVersion: cert-manager.io/v1 kind: Certificate @@ -48,6 +48,7 @@ spec: encoding: PKCS8 size: 256 usages: + - client auth - server auth dnsNames: - "lokistack-dev-gateway-http.default.svc" @@ -58,6 +59,27 @@ spec: --- apiVersion: cert-manager.io/v1 kind: Certificate +metadata: + name: lokistack-dev-gateway-client-http + namespace: default +spec: + secretName: lokistack-dev-gateway-client-http + duration: 2160h # 90d + renewBefore: 360h # 15d + commonName: lokistack-dev-gateway-http.default.svc + privateKey: + rotationPolicy: Never + algorithm: ECDSA + encoding: PKCS8 + size: 256 + usages: + - client auth + issuerRef: + name: kind-issuer + kind: Issuer +--- +apiVersion: cert-manager.io/v1 +kind: Certificate metadata: name: lokistack-dev-distributor-grpc namespace: default @@ -72,6 +94,7 @@ spec: encoding: PKCS8 size: 256 usages: + - client auth - server auth dnsNames: - "lokistack-dev-distributor-grpc.default.svc" @@ -96,6 +119,7 @@ spec: encoding: PKCS8 size: 256 usages: + - client auth - server auth dnsNames: - "lokistack-dev-distributor-http.default.svc" @@ -120,6 +144,7 @@ spec: encoding: PKCS8 size: 256 usages: + - client auth - server auth dnsNames: - "lokistack-dev-ingester-grpc.default.svc" @@ -144,6 +169,7 @@ spec: encoding: PKCS8 size: 256 usages: + - client auth - server auth dnsNames: - "lokistack-dev-ingester-http.default.svc" @@ -168,6 +194,7 @@ spec: encoding: PKCS8 size: 256 usages: + - client auth - server auth dnsNames: - "lokistack-dev-query-frontend-grpc.default.svc" @@ -192,6 +219,7 @@ spec: encoding: PKCS8 size: 256 usages: + - client auth - server auth dnsNames: - "lokistack-dev-query-frontend-http.default.svc" @@ -216,6 +244,7 @@ spec: encoding: PKCS8 size: 256 usages: + - client auth - server auth dnsNames: - "lokistack-dev-querier-grpc.default.svc" @@ -240,6 +269,7 @@ spec: encoding: PKCS8 size: 256 usages: + - client auth - server auth dnsNames: - "lokistack-dev-querier-http.default.svc" @@ -264,6 +294,7 @@ spec: encoding: PKCS8 size: 256 usages: + - client auth - server auth dnsNames: - "lokistack-dev-index-gateway-grpc.default.svc" @@ -288,6 +319,7 @@ spec: encoding: PKCS8 size: 256 usages: + - client auth - server auth dnsNames: - "lokistack-dev-index-gateway-http.default.svc" @@ -312,6 +344,7 @@ spec: encoding: PKCS8 size: 256 usages: + - client auth - server auth dnsNames: - "lokistack-dev-compactor-grpc.default.svc" @@ -336,6 +369,7 @@ spec: encoding: PKCS8 size: 256 usages: + - client auth - server auth dnsNames: - "lokistack-dev-compactor-http.default.svc" diff --git a/operator/internal/certrotation/build.go b/operator/internal/certrotation/build.go new file mode 100644 index 000000000000..4f85d3935e44 --- /dev/null +++ b/operator/internal/certrotation/build.go @@ -0,0 +1,76 @@ +package certrotation + +import ( + "fmt" + "time" + + configv1 "github.com/grafana/loki/operator/apis/config/v1" + "k8s.io/apiserver/pkg/authentication/user" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var defaultUserInfo = &user.DefaultInfo{Name: "system:lokistacks", Groups: []string{"system:logging"}} + +// BuildAll builds all secrets and configmaps containing +// CA certificates, CA bundles and client certificates for +// a LokiStack. +func BuildAll(opts Options) ([]client.Object, error) { + res := make([]client.Object, 0) + + obj, err := buildSigningCASecret(&opts) + if err != nil { + return nil, err + } + res = append(res, obj) + + obj, err = buildCABundle(&opts) + if err != nil { + return nil, err + } + res = append(res, obj) + + objs, err := buildTargetCertKeyPairSecrets(opts) + if err != nil { + return nil, err + } + res = append(res, objs...) + + return res, nil +} + +// ApplyDefaultSettings merges the default options with the ones we give. +func ApplyDefaultSettings(opts *Options, cfg configv1.BuiltInCertManagement) error { + rotation, err := ParseRotation(cfg) + if err != nil { + return err + } + opts.Rotation = rotation + + clock := time.Now + opts.Signer.Rotation = signerRotation{ + Clock: clock, + } + + if opts.Certificates == nil { + opts.Certificates = make(map[string]SelfSignedCertKey) + } + for _, name := range ComponentCertSecretNames(opts.StackName) { + r := certificateRotation{ + Clock: clock, + UserInfo: defaultUserInfo, + Hostnames: []string{ + fmt.Sprintf("%s.%s.svc", name, opts.StackNamespace), + fmt.Sprintf("%s.%s.svc.cluster.local", name, opts.StackNamespace), + }, + } + + cert, ok := opts.Certificates[name] + if !ok { + cert = SelfSignedCertKey{} + } + cert.Rotation = r + opts.Certificates[name] = cert + } + + return nil +} diff --git a/operator/internal/certrotation/build_test.go b/operator/internal/certrotation/build_test.go new file mode 100644 index 000000000000..dd57ad183e62 --- /dev/null +++ b/operator/internal/certrotation/build_test.go @@ -0,0 +1,134 @@ +package certrotation + +import ( + "fmt" + "strings" + "testing" + + configv1 "github.com/grafana/loki/operator/apis/config/v1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestBuildAll(t *testing.T) { + cfg := configv1.BuiltInCertManagement{ + CACertValidity: "10m", + CACertRefresh: "5m", + CertValidity: "2m", + CertRefresh: "1m", + } + + opts := Options{ + StackName: "dev", + StackNamespace: "ns", + } + err := ApplyDefaultSettings(&opts, cfg) + require.NoError(t, err) + + objs, err := BuildAll(opts) + require.NoError(t, err) + require.Len(t, objs, 17) + + for _, obj := range objs { + require.True(t, strings.HasPrefix(obj.GetName(), opts.StackName)) + require.Equal(t, obj.GetNamespace(), opts.StackNamespace) + + switch o := obj.(type) { + case *corev1.Secret: + require.Contains(t, o.Annotations, CertificateIssuer) + require.Contains(t, o.Annotations, CertificateNotAfterAnnotation) + require.Contains(t, o.Annotations, CertificateNotBeforeAnnotation) + } + } +} + +func TestApplyDefaultSettings_EmptySecrets(t *testing.T) { + cfg := configv1.BuiltInCertManagement{ + CACertValidity: "10m", + CACertRefresh: "5m", + CertValidity: "2m", + CertRefresh: "1m", + } + + opts := Options{ + StackName: "lokistack-dev", + StackNamespace: "ns", + } + + err := ApplyDefaultSettings(&opts, cfg) + require.NoError(t, err) + + cs := ComponentCertSecretNames(opts.StackName) + + for _, name := range cs { + cert, ok := opts.Certificates[name] + require.True(t, ok) + require.NotEmpty(t, cert.Rotation) + + hostnames := []string{ + fmt.Sprintf("%s.%s.svc", name, opts.StackNamespace), + fmt.Sprintf("%s.%s.svc.cluster.local", name, opts.StackNamespace), + } + + require.ElementsMatch(t, hostnames, cert.Rotation.Hostnames) + require.Equal(t, defaultUserInfo, cert.Rotation.UserInfo) + require.Nil(t, cert.Secret) + } +} + +func TestApplyDefaultSettings_ExistingSecrets(t *testing.T) { + const ( + stackName = "dev" + stackNamespace = "ns" + ) + + cfg := configv1.BuiltInCertManagement{ + CACertValidity: "10m", + CACertRefresh: "5m", + CertValidity: "2m", + CertRefresh: "1m", + } + + opts := Options{ + StackName: stackName, + StackNamespace: stackNamespace, + Certificates: ComponentCertificates{}, + } + + cs := ComponentCertSecretNames(opts.StackName) + + for _, name := range cs { + opts.Certificates[name] = SelfSignedCertKey{ + Secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: stackNamespace, + Annotations: map[string]string{ + CertificateNotBeforeAnnotation: "not-before", + CertificateNotAfterAnnotation: "not-after", + }, + }, + }, + } + } + + err := ApplyDefaultSettings(&opts, cfg) + require.NoError(t, err) + + for _, name := range cs { + cert, ok := opts.Certificates[name] + require.True(t, ok) + require.NotEmpty(t, cert.Rotation) + + hostnames := []string{ + fmt.Sprintf("%s.%s.svc", name, opts.StackNamespace), + fmt.Sprintf("%s.%s.svc.cluster.local", name, opts.StackNamespace), + } + + require.ElementsMatch(t, hostnames, cert.Rotation.Hostnames) + require.Equal(t, defaultUserInfo, cert.Rotation.UserInfo) + + require.NotNil(t, cert.Secret) + } +} diff --git a/operator/internal/certrotation/cabundle.go b/operator/internal/certrotation/cabundle.go new file mode 100644 index 000000000000..319bfd8d5cce --- /dev/null +++ b/operator/internal/certrotation/cabundle.go @@ -0,0 +1,86 @@ +package certrotation + +import ( + "bytes" + "crypto/x509" + + "github.com/openshift/library-go/pkg/crypto" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/cert" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// buildCABundle returns a ConfigMap including all known non-expired signing CAs across rotations. +func buildCABundle(opts *Options) (client.Object, error) { + cm := newConfigMap(*opts) + + certs, err := manageCABundleConfigMap(cm, opts.Signer.RawCA.Config.Certs[0]) + if err != nil { + return nil, err + } + + opts.RawCACerts = certs + + caBytes, err := crypto.EncodeCertificates(certs...) + if err != nil { + return nil, err + } + + cm.Data[CAFile] = string(caBytes) + + return cm, nil +} + +func newConfigMap(opts Options) *corev1.ConfigMap { + current := opts.CABundle.DeepCopy() + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: CABundleName(opts.StackName), + Namespace: opts.StackNamespace, + }, + } + + if current != nil { + cm.Annotations = current.Annotations + cm.Labels = current.Labels + cm.Data = current.Data + } + + return cm +} + +// manageCABundleConfigMap adds the new certificate to the list of cabundles, eliminates duplicates, and prunes the list of expired +// certs to trust as signers +func manageCABundleConfigMap(caBundleConfigMap *corev1.ConfigMap, currentSigner *x509.Certificate) ([]*x509.Certificate, error) { + if caBundleConfigMap.Data == nil { + caBundleConfigMap.Data = map[string]string{} + } + + certificates := []*x509.Certificate{} + caBundle := caBundleConfigMap.Data[CAFile] + if len(caBundle) > 0 { + var err error + certificates, err = cert.ParseCertsPEM([]byte(caBundle)) + if err != nil { + return nil, err + } + } + certificates = append([]*x509.Certificate{currentSigner}, certificates...) + certificates = crypto.FilterExpiredCerts(certificates...) + + finalCertificates := []*x509.Certificate{} + // now check for duplicates. n^2, but super simple +nextCertificate: + for i := range certificates { + for j := range finalCertificates { + if bytes.Equal(certificates[i].Raw, finalCertificates[j].Raw) { + continue nextCertificate + } + } + finalCertificates = append(finalCertificates, certificates[i]) + } + + return finalCertificates, nil +} diff --git a/operator/internal/certrotation/cabundle_test.go b/operator/internal/certrotation/cabundle_test.go new file mode 100644 index 000000000000..2e8ecbc1324e --- /dev/null +++ b/operator/internal/certrotation/cabundle_test.go @@ -0,0 +1,74 @@ +package certrotation + +import ( + "bytes" + "testing" + "time" + + "github.com/openshift/library-go/pkg/crypto" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestBuildCABundle_Create(t *testing.T) { + rawCA, _ := newTestCABundle(t, "test-ca") + + opts := &Options{ + StackName: "dev", + StackNamespace: "ns", + Signer: SigningCA{ + RawCA: rawCA, + }, + } + + obj, err := buildCABundle(opts) + require.NoError(t, err) + require.NotNil(t, obj) + require.Len(t, opts.RawCACerts, 1) +} + +func TestBuildCABundle_Append(t *testing.T) { + _, rawCABytes := newTestCABundle(t, "test-ca") + newRawCA, _ := newTestCABundle(t, "test-ca-other") + + opts := &Options{ + StackName: "dev", + StackNamespace: "ns", + Signer: SigningCA{ + RawCA: newRawCA, + }, + CABundle: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dev-ca-bundle", + Namespace: "ns", + }, + Data: map[string]string{ + CAFile: string(rawCABytes), + }, + }, + } + + obj, err := buildCABundle(opts) + require.NoError(t, err) + require.NotNil(t, obj) + require.Len(t, opts.RawCACerts, 2) +} + +func newTestCABundle(t *testing.T, name string) (*crypto.CA, []byte) { + testCA, err := crypto.MakeSelfSignedCAConfigForDuration(name, 1*time.Hour) + require.NoError(t, err) + + certBytes := &bytes.Buffer{} + keyBytes := &bytes.Buffer{} + err = testCA.WriteCertConfig(certBytes, keyBytes) + require.NoError(t, err) + + rawCA, err := crypto.GetCAFromBytes(certBytes.Bytes(), keyBytes.Bytes()) + require.NoError(t, err) + + rawCABytes, err := crypto.EncodeCertificates(rawCA.Config.Certs...) + require.NoError(t, err) + + return rawCA, rawCABytes +} diff --git a/operator/internal/certrotation/expiry.go b/operator/internal/certrotation/expiry.go new file mode 100644 index 000000000000..aaa2bb6a75a7 --- /dev/null +++ b/operator/internal/certrotation/expiry.go @@ -0,0 +1,17 @@ +package certrotation + +import ( + "fmt" + "strings" +) + +// CertExpiredError contains information if a certificate expired +// and the reasons of expiry. +type CertExpiredError struct { + Message string + Reasons []string +} + +func (e *CertExpiredError) Error() string { + return fmt.Sprintf("%s for reasons: %s", e.Message, strings.Join(e.Reasons, ", ")) +} diff --git a/operator/internal/certrotation/options.go b/operator/internal/certrotation/options.go new file mode 100644 index 000000000000..4b30ae7889ff --- /dev/null +++ b/operator/internal/certrotation/options.go @@ -0,0 +1,86 @@ +package certrotation + +import ( + "crypto/x509" + "time" + + "github.com/ViaQ/logerr/v2/kverrors" + configv1 "github.com/grafana/loki/operator/apis/config/v1" + "github.com/openshift/library-go/pkg/crypto" + corev1 "k8s.io/api/core/v1" +) + +// ComponentCertificates is a map of lokistack component names to TLS certificates +type ComponentCertificates map[string]SelfSignedCertKey + +// Options is a set of configuration values to use when +// building manifests for LokiStack certificates. +type Options struct { + StackName string + StackNamespace string + Rotation Rotation + Signer SigningCA + CABundle *corev1.ConfigMap + RawCACerts []*x509.Certificate + Certificates ComponentCertificates +} + +// SigningCA rotates a self-signed signing CA stored in a secret. It creates a new one when +// - refresh duration is over +// - or 80% of validity is over +// - or the CA is expired. +type SigningCA struct { + RawCA *crypto.CA + Secret *corev1.Secret + Rotation signerRotation +} + +// SelfSignedCertKey rotates a key and cert signed by a signing CA and stores it in a secret. +// +// It creates a new one when +// - refresh duration is over +// - or 80% of validity is over +// - or the cert is expired. +// - or the signing CA changes. +type SelfSignedCertKey struct { + Secret *corev1.Secret + Rotation certificateRotation +} + +// Rotation define the validity/refresh pairs for certificates +type Rotation struct { + CACertValidity time.Duration + CACertRefresh time.Duration + TargetCertValidity time.Duration + TargetCertRefresh time.Duration +} + +// ParseRotation builds a new RotationOptions struct from the feature gate string values. +func ParseRotation(cfg configv1.BuiltInCertManagement) (Rotation, error) { + caValidity, err := time.ParseDuration(cfg.CACertValidity) + if err != nil { + return Rotation{}, kverrors.Wrap(err, "failed to parse CA validity duration", "value", cfg.CACertValidity) + } + + caRefresh, err := time.ParseDuration(cfg.CACertRefresh) + if err != nil { + return Rotation{}, kverrors.Wrap(err, "failed to parse CA refresh duration", "value", cfg.CACertRefresh) + } + + certValidity, err := time.ParseDuration(cfg.CertValidity) + if err != nil { + return Rotation{}, kverrors.Wrap(err, "failed to parse target certificate validity duration", "value", cfg.CertValidity) + } + + certRefresh, err := time.ParseDuration(cfg.CertRefresh) + if err != nil { + return Rotation{}, kverrors.Wrap(err, "failed to parse target certificate refresh duration", "value", cfg.CertRefresh) + } + + return Rotation{ + CACertValidity: caValidity, + CACertRefresh: caRefresh, + TargetCertValidity: certValidity, + TargetCertRefresh: certRefresh, + }, nil +} diff --git a/operator/internal/certrotation/rotation.go b/operator/internal/certrotation/rotation.go new file mode 100644 index 000000000000..e2913cec4e88 --- /dev/null +++ b/operator/internal/certrotation/rotation.go @@ -0,0 +1,184 @@ +package certrotation + +import ( + "crypto/x509" + "crypto/x509/pkix" + "errors" + "fmt" + "strings" + "time" + + "github.com/openshift/library-go/pkg/certs" + "github.com/openshift/library-go/pkg/crypto" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/authentication/user" +) + +var ( + errMissingIssuer = errors.New("no issuer set") + errMissingHostnames = errors.New("no hostnames set") + errMissingUserInfo = errors.New("no user info") +) + +type clockFunc func() time.Time + +type signerRotation struct { + Issuer string + Clock clockFunc +} + +func (r *signerRotation) NewCertificate(validity time.Duration) (*crypto.TLSCertificateConfig, error) { + if r.Issuer == "" { + return nil, errMissingIssuer + } + + signerName := fmt.Sprintf("%s@%d", r.Issuer, time.Now().Unix()) + return crypto.MakeSelfSignedCAConfigForDuration(signerName, validity) +} + +func (r *signerRotation) NeedNewCertificate(annotations map[string]string, refresh time.Duration) string { + return needNewCertificate(annotations, r.Clock, refresh, nil) +} + +func (r *signerRotation) SetAnnotations(ca *crypto.TLSCertificateConfig, annotations map[string]string) { + annotations[CertificateNotAfterAnnotation] = ca.Certs[0].NotAfter.Format(time.RFC3339) + annotations[CertificateNotBeforeAnnotation] = ca.Certs[0].NotBefore.Format(time.RFC3339) + annotations[CertificateIssuer] = ca.Certs[0].Issuer.CommonName +} + +type certificateRotation struct { + UserInfo user.Info + Hostnames []string + Clock clockFunc +} + +func (r *certificateRotation) NewCertificate(signer *crypto.CA, validity time.Duration) (*crypto.TLSCertificateConfig, error) { + if r.UserInfo == nil { + return nil, errMissingUserInfo + } + if len(r.Hostnames) == 0 { + return nil, errMissingHostnames + } + + addClientAuthUsage := func(cert *x509.Certificate) error { + cert.ExtKeyUsage = append(cert.ExtKeyUsage, x509.ExtKeyUsageClientAuth) + return nil + } + + addSubject := func(cert *x509.Certificate) error { + cert.Subject = pkix.Name{ + CommonName: r.UserInfo.GetName(), + SerialNumber: r.UserInfo.GetUID(), + Organization: r.UserInfo.GetGroups(), + } + return nil + } + + return signer.MakeServerCertForDuration(sets.NewString(r.Hostnames...), validity, addClientAuthUsage, addSubject) +} + +func (r *certificateRotation) NeedNewCertificate(annotations map[string]string, signer *crypto.CA, caBundleCerts []*x509.Certificate, refresh time.Duration) string { + reason := needNewCertificate(annotations, r.Clock, refresh, signer) + if len(reason) > 0 { + return reason + } + + // check the signer common name against all the common names in our ca bundle so we don't refresh early + signerCommonName := annotations[CertificateIssuer] + if signerCommonName == "" { + return "missing issuer name" + } + + var found bool + for _, caCert := range caBundleCerts { + if signerCommonName == caCert.Subject.CommonName { + found = true + break + } + } + + if !found { + return fmt.Sprintf("issuer %q, not in ca bundle:\n%s", signerCommonName, certs.CertificateBundleToString(caBundleCerts)) + } + + existingHostnames := sets.NewString(strings.Split(annotations[CertificateHostnames], ",")...) + requiredHostnames := sets.NewString(r.Hostnames...) + if !existingHostnames.Equal(requiredHostnames) { + existingNotRequired := existingHostnames.Difference(requiredHostnames) + requiredNotExisting := requiredHostnames.Difference(existingHostnames) + return fmt.Sprintf("hostnames %q are existing and not required, %q are required and not existing", strings.Join(existingNotRequired.List(), ","), strings.Join(requiredNotExisting.List(), ",")) + } + + return "" +} + +func (r *certificateRotation) SetAnnotations(cert *crypto.TLSCertificateConfig, annotations map[string]string) { + hostnames := sets.String{} + for _, ip := range cert.Certs[0].IPAddresses { + hostnames.Insert(ip.String()) + } + for _, dnsName := range cert.Certs[0].DNSNames { + hostnames.Insert(dnsName) + } + + annotations[CertificateNotAfterAnnotation] = cert.Certs[0].NotAfter.Format(time.RFC3339) + annotations[CertificateNotBeforeAnnotation] = cert.Certs[0].NotBefore.Format(time.RFC3339) + annotations[CertificateIssuer] = cert.Certs[0].Issuer.CommonName + // List does a sort so that we have a consistent representation + annotations[CertificateHostnames] = strings.Join(hostnames.List(), ",") +} + +func needNewCertificate(annotations map[string]string, clock clockFunc, refresh time.Duration, signer *crypto.CA) string { + notAfterString := annotations[CertificateNotAfterAnnotation] + if len(notAfterString) == 0 { + return "missing notAfter" + } + notAfter, err := time.Parse(time.RFC3339, notAfterString) + if err != nil { + return fmt.Sprintf("bad expiry: %q", notAfterString) + } + + notBeforeString := annotations[CertificateNotBeforeAnnotation] + if len(notAfterString) == 0 { + return "missing notBefore" + } + notBefore, err := time.Parse(time.RFC3339, notBeforeString) + if err != nil { + return fmt.Sprintf("bad expiry: %q", notBeforeString) + } + + now := clock() + + // Is cert expired? + if now.After(notAfter) { + return "already expired" + } + + // Refresh only when expired + validity := notAfter.Sub(notBefore) + if validity == refresh { + return "" + } + + // Are we at 80% of validity? + at80Percent := notAfter.Add(-validity / 5) + if now.After(at80Percent) { + return fmt.Sprintf("past its latest possible time %v", at80Percent) + } + + // If Certificate is past its refresh time, we may have action to take. We only do this if the signer is old enough. + developerSpecifiedRefresh := notBefore.Add(refresh) + if now.After(developerSpecifiedRefresh) { + if signer == nil { + return fmt.Sprintf("past its refresh time %v", developerSpecifiedRefresh) + } + + // make sure the signer has been valid for more than 10% of the target's refresh time. + timeToWaitForTrustRotation := refresh / 10 + if now.After(signer.Config.Certs[0].NotBefore.Add(timeToWaitForTrustRotation)) { + return fmt.Sprintf("past its refresh time %v", developerSpecifiedRefresh) + } + } + + return "" +} diff --git a/operator/internal/certrotation/rotation_test.go b/operator/internal/certrotation/rotation_test.go new file mode 100644 index 000000000000..89f17225ced8 --- /dev/null +++ b/operator/internal/certrotation/rotation_test.go @@ -0,0 +1,335 @@ +package certrotation + +import ( + stdcrypto "crypto" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "math/big" + "testing" + "time" + + "github.com/openshift/library-go/pkg/crypto" + "github.com/stretchr/testify/require" +) + +func TestSignerRotation_ReturnErrorOnMissingIssuer(t *testing.T) { + c := signerRotation{} + _, err := c.NewCertificate(1 * time.Hour) + require.ErrorIs(t, err, errMissingIssuer) +} + +func TestSignerRotation_SetAnnotations(t *testing.T) { + var ( + now = time.Now() + nowFn = func() time.Time { return now } + nowCA, err = newTestCACertificate(pkix.Name{CommonName: "creator-tests"}, int64(1), 200*time.Minute, nowFn) + ) + require.NoError(t, err) + + c := signerRotation{} + + annotations := map[string]string{} + c.SetAnnotations(nowCA.Config, annotations) + + require.Len(t, annotations, 3) + require.Contains(t, annotations, CertificateIssuer) + require.Contains(t, annotations, CertificateNotBeforeAnnotation) + require.Contains(t, annotations, CertificateNotAfterAnnotation) +} + +func TestSignerRotation_NeedNewCertificate(t *testing.T) { + var ( + now = time.Now() + nowFn = func() time.Time { return now } + invalidNotAfter, _ = time.Parse(time.RFC3339, "") + invalidNotBefore, _ = time.Parse(time.RFC3339, "") + ) + + tt := []struct { + desc string + annotations map[string]string + refresh time.Duration + wantReason string + }{ + { + desc: "already expired", + annotations: map[string]string{ + CertificateIssuer: "creator-tests", + CertificateNotAfterAnnotation: invalidNotAfter.Format(time.RFC3339), + CertificateNotBeforeAnnotation: invalidNotBefore.Format(time.RFC3339), + }, + refresh: 2 * time.Minute, + wantReason: "already expired", + }, + { + desc: "refresh only when expired", + annotations: map[string]string{ + CertificateIssuer: "creator-tests", + CertificateNotAfterAnnotation: now.Add(45 * time.Minute).Format(time.RFC3339), + CertificateNotBeforeAnnotation: now.Add(-45 * time.Minute).Format(time.RFC3339), + }, + refresh: 90 * time.Minute, + }, + { + desc: "at 80 percent validity", + annotations: map[string]string{ + CertificateIssuer: "creator-tests", + CertificateNotAfterAnnotation: now.Add(18 * time.Minute).Format(time.RFC3339), + CertificateNotBeforeAnnotation: now.Add(-72 * time.Minute).Format(time.RFC3339), + }, + refresh: 40 * time.Minute, + wantReason: "past its latest possible time", + }, + { + desc: "past its refresh time", + annotations: map[string]string{ + CertificateIssuer: "creator-tests", + CertificateNotAfterAnnotation: now.Add(45 * time.Minute).Format(time.RFC3339), + CertificateNotBeforeAnnotation: now.Add(-45 * time.Minute).Format(time.RFC3339), + }, + refresh: 40 * time.Minute, + wantReason: "past its refresh time", + }, + } + for _, tc := range tt { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + c := signerRotation{Clock: nowFn} + reason := c.NeedNewCertificate(tc.annotations, tc.refresh) + require.Contains(t, reason, tc.wantReason) + }) + } +} + +func TestCertificateRotation_ReturnErrorOnMissingUserInfo(t *testing.T) { + c := certificateRotation{} + _, err := c.NewCertificate(nil, 1*time.Hour) + require.ErrorIs(t, err, errMissingUserInfo) +} + +func TestCertificateRotation_ReturnErrorOnMissingHostnames(t *testing.T) { + c := certificateRotation{UserInfo: defaultUserInfo} + _, err := c.NewCertificate(nil, 1*time.Hour) + require.ErrorIs(t, err, errMissingHostnames) +} + +func TestCertificateRotation_CertHasRequiredExtensions(t *testing.T) { + var ( + now = time.Now() + nowFn = func() time.Time { return now } + nowCA, err = newTestCACertificate(pkix.Name{CommonName: "creator-tests"}, int64(1), 200*time.Minute, nowFn) + ) + require.NoError(t, err) + + c := certificateRotation{ + UserInfo: defaultUserInfo, + Hostnames: []string{"example.org"}, + } + cert, err := c.NewCertificate(nowCA, 1*time.Hour) + require.NoError(t, err) + + require.Contains(t, cert.Certs[0].ExtKeyUsage, x509.ExtKeyUsageServerAuth) + require.Contains(t, cert.Certs[0].ExtKeyUsage, x509.ExtKeyUsageClientAuth) + require.Equal(t, defaultUserInfo.GetName(), cert.Certs[0].Subject.CommonName) + require.Equal(t, defaultUserInfo.GetUID(), cert.Certs[0].Subject.SerialNumber) + require.Equal(t, defaultUserInfo.GetGroups(), cert.Certs[0].Subject.Organization) +} + +func TestCertificateRotation_SetAnnotations(t *testing.T) { + var ( + now = time.Now() + nowFn = func() time.Time { return now } + nowCA, err = newTestCACertificate(pkix.Name{CommonName: "creator-tests"}, int64(1), 200*time.Minute, nowFn) + ) + require.NoError(t, err) + + c := certificateRotation{Hostnames: []string{"example.org"}} + + annotations := map[string]string{} + c.SetAnnotations(nowCA.Config, annotations) + + require.Len(t, annotations, 4) + require.Contains(t, annotations, CertificateIssuer) + require.Contains(t, annotations, CertificateNotBeforeAnnotation) + require.Contains(t, annotations, CertificateNotAfterAnnotation) + require.Contains(t, annotations, CertificateHostnames) +} + +func TestCertificateRotation_NeedNewCertificate(t *testing.T) { + var ( + now = time.Now() + nowFn = func() time.Time { return now } + invalidNotAfter, _ = time.Parse(time.RFC3339, "") + invalidNotBefore, _ = time.Parse(time.RFC3339, "") + nowCA, _ = newTestCACertificate(pkix.Name{CommonName: "creator-tests"}, int64(1), 200*time.Minute, nowFn) + + twentyMinutesBeforeNow = time.Now().Add(-20 * time.Minute) + twentyMinutesBeforeNowFn = func() time.Time { return twentyMinutesBeforeNow } + twentyMinutesBeforeCA, _ = newTestCACertificate(pkix.Name{CommonName: "creator-tests"}, int64(1), 200*time.Minute, twentyMinutesBeforeNowFn) + ) + + tt := []struct { + desc string + annotations map[string]string + signerFn func() (*crypto.CA, error) + refresh time.Duration + wantReason string + }{ + { + desc: "already expired", + annotations: map[string]string{ + CertificateIssuer: "creator-tests", + CertificateNotAfterAnnotation: invalidNotAfter.Format(time.RFC3339), + CertificateNotBeforeAnnotation: invalidNotBefore.Format(time.RFC3339), + }, + signerFn: func() (*crypto.CA, error) { + return nowCA, nil + }, + refresh: 2 * time.Minute, + wantReason: "already expired", + }, + { + desc: "refresh only when expired", + annotations: map[string]string{ + CertificateIssuer: "creator-tests", + CertificateNotAfterAnnotation: now.Add(45 * time.Minute).Format(time.RFC3339), + CertificateNotBeforeAnnotation: now.Add(-45 * time.Minute).Format(time.RFC3339), + }, + signerFn: func() (*crypto.CA, error) { + return nowCA, nil + }, + refresh: 90 * time.Minute, + }, + { + desc: "at 80 percent validity", + annotations: map[string]string{ + CertificateIssuer: "creator-tests", + CertificateNotAfterAnnotation: now.Add(18 * time.Minute).Format(time.RFC3339), + CertificateNotBeforeAnnotation: now.Add(-72 * time.Minute).Format(time.RFC3339), + }, + signerFn: func() (*crypto.CA, error) { + return nowCA, nil + }, + refresh: 40 * time.Minute, + wantReason: "past its latest possible time", + }, + { + desc: "past its refresh time", + annotations: map[string]string{ + CertificateIssuer: "creator-tests", + CertificateNotAfterAnnotation: now.Add(45 * time.Minute).Format(time.RFC3339), + CertificateNotBeforeAnnotation: now.Add(-45 * time.Minute).Format(time.RFC3339), + }, + signerFn: func() (*crypto.CA, error) { + return twentyMinutesBeforeCA, nil + }, + refresh: 40 * time.Minute, + wantReason: "past its refresh time", + }, + { + desc: "missing issuer name", + annotations: map[string]string{ + CertificateNotAfterAnnotation: now.Add(45 * time.Minute).Format(time.RFC3339), + CertificateNotBeforeAnnotation: now.Add(-45 * time.Minute).Format(time.RFC3339), + }, + signerFn: func() (*crypto.CA, error) { + return nowCA, nil + }, + refresh: 70 * time.Minute, + wantReason: "missing issuer name", + }, + { + desc: "issuer not in ca bundle", + annotations: map[string]string{ + CertificateIssuer: "issuer-not-in-any-ca", + CertificateNotAfterAnnotation: now.Add(45 * time.Minute).Format(time.RFC3339), + CertificateNotBeforeAnnotation: now.Add(-45 * time.Minute).Format(time.RFC3339), + }, + signerFn: func() (*crypto.CA, error) { + return nowCA, nil + }, + refresh: 70 * time.Minute, + wantReason: `issuer "issuer-not-in-any-ca", not in ca bundle`, + }, + { + desc: "missing hostnames", + annotations: map[string]string{ + CertificateIssuer: "creator-tests", + CertificateNotAfterAnnotation: now.Add(45 * time.Minute).Format(time.RFC3339), + CertificateNotBeforeAnnotation: now.Add(-45 * time.Minute).Format(time.RFC3339), + }, + signerFn: func() (*crypto.CA, error) { + return nowCA, nil + }, + refresh: 70 * time.Minute, + wantReason: "are required and not existing", + }, + } + for _, tc := range tt { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + rawCA, err := tc.signerFn() + require.NoError(t, err) + + c := certificateRotation{ + Clock: nowFn, + Hostnames: []string{"a.b.c.d", "e.d.f.g"}, + } + reason := c.NeedNewCertificate(tc.annotations, rawCA, rawCA.Config.Certs, tc.refresh) + require.Contains(t, reason, tc.wantReason) + }) + } +} + +func newTestCACertificate(subject pkix.Name, serialNumber int64, validity time.Duration, currentTime func() time.Time) (*crypto.CA, error) { + caPublicKey, caPrivateKey, err := crypto.NewKeyPair() + if err != nil { + return nil, err + } + + caCert := &x509.Certificate{ + Subject: subject, + + SignatureAlgorithm: x509.SHA256WithRSA, + + NotBefore: currentTime().Add(-1 * time.Second), + NotAfter: currentTime().Add(validity), + SerialNumber: big.NewInt(serialNumber), + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + IsCA: true, + } + + cert, err := signCertificate(caCert, caPublicKey, caCert, caPrivateKey) + if err != nil { + return nil, err + } + + return &crypto.CA{ + Config: &crypto.TLSCertificateConfig{ + Certs: []*x509.Certificate{cert}, + Key: caPrivateKey, + }, + SerialGenerator: &crypto.RandomSerialGenerator{}, + }, nil +} + +func signCertificate(template *x509.Certificate, requestKey stdcrypto.PublicKey, issuer *x509.Certificate, issuerKey stdcrypto.PrivateKey) (*x509.Certificate, error) { + derBytes, err := x509.CreateCertificate(rand.Reader, template, issuer, requestKey, issuerKey) + if err != nil { + return nil, err + } + certs, err := x509.ParseCertificates(derBytes) + if err != nil { + return nil, err + } + if len(certs) != 1 { + return nil, errors.New("Expected a single certificate") + } + return certs[0], nil +} diff --git a/operator/internal/certrotation/signer.go b/operator/internal/certrotation/signer.go new file mode 100644 index 000000000000..244c18b94a30 --- /dev/null +++ b/operator/internal/certrotation/signer.go @@ -0,0 +1,100 @@ +package certrotation + +import ( + "bytes" + "fmt" + "time" + + "github.com/openshift/library-go/pkg/crypto" + "sigs.k8s.io/controller-runtime/pkg/client" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// SigningCAExpired returns true if the signer certificate expired and the reason of expiry. +func SigningCAExpired(opts Options) error { + // Skip as secret not created or loaded + if opts.Signer.Secret == nil { + return nil + } + + reason := opts.Signer.Rotation.NeedNewCertificate(opts.Signer.Secret.Annotations, opts.Rotation.CACertRefresh) + if reason != "" { + return &CertExpiredError{Message: "signing CA certificate expired", Reasons: []string{reason}} + } + + return nil +} + +// buildSigningCASecret returns a k8s Secret holding the signing CA certificate +func buildSigningCASecret(opts *Options) (client.Object, error) { + signingCertKeyPairSecret := newSigningCASecret(*opts) + opts.Signer.Rotation.Issuer = fmt.Sprintf("%s_%s", signingCertKeyPairSecret.Namespace, signingCertKeyPairSecret.Name) + + if reason := opts.Signer.Rotation.NeedNewCertificate(signingCertKeyPairSecret.Annotations, opts.Rotation.CACertRefresh); reason != "" { + if err := setSigningCertKeyPairSecret(signingCertKeyPairSecret, opts.Rotation.CACertValidity, opts.Signer.Rotation); err != nil { + return nil, err + } + } + + var ( + cert = signingCertKeyPairSecret.Data[corev1.TLSCertKey] + key = signingCertKeyPairSecret.Data[corev1.TLSPrivateKeyKey] + ) + + rawCA, err := crypto.GetCAFromBytes(cert, key) + if err != nil { + return nil, err + } + + opts.Signer.RawCA = rawCA + + return signingCertKeyPairSecret, nil +} + +func newSigningCASecret(opts Options) *corev1.Secret { + current := opts.Signer.Secret.DeepCopy() + + s := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: SigningCASecretName(opts.StackName), + Namespace: opts.StackNamespace, + }, + Type: corev1.SecretTypeTLS, + } + + if current != nil { + s.Annotations = current.Annotations + s.Labels = current.Labels + s.Data = current.Data + } + + return s +} + +// setSigningCertKeyPairSecret creates a new signing cert/key pair and sets them in the secret +func setSigningCertKeyPairSecret(s *corev1.Secret, validity time.Duration, caCreator signerRotation) error { + if s.Annotations == nil { + s.Annotations = map[string]string{} + } + if s.Data == nil { + s.Data = map[string][]byte{} + } + + ca, err := caCreator.NewCertificate(validity) + if err != nil { + return err + } + + certBytes := &bytes.Buffer{} + keyBytes := &bytes.Buffer{} + if err := ca.WriteCertConfig(certBytes, keyBytes); err != nil { + return err + } + s.Data[corev1.TLSCertKey] = certBytes.Bytes() + s.Data[corev1.TLSPrivateKeyKey] = keyBytes.Bytes() + caCreator.SetAnnotations(ca, s.Annotations) + + return nil +} diff --git a/operator/internal/certrotation/signer_test.go b/operator/internal/certrotation/signer_test.go new file mode 100644 index 000000000000..8b4778053ee0 --- /dev/null +++ b/operator/internal/certrotation/signer_test.go @@ -0,0 +1,131 @@ +package certrotation + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestSigningCAExpired_EmptySecret(t *testing.T) { + opts := Options{ + StackName: "dev", + StackNamespace: "ns", + } + + err := SigningCAExpired(opts) + require.NoError(t, err) +} + +func TestSigningCAExpired_ExpiredSecret(t *testing.T) { + var ( + stackName = "dev" + stackNamespace = "ns" + clock = time.Now + invalidNotAfter, _ = time.Parse(time.RFC3339, "") + invalidNotBefore, _ = time.Parse(time.RFC3339, "") + ) + + opts := Options{ + StackName: stackName, + StackNamespace: stackNamespace, + Signer: SigningCA{ + Rotation: signerRotation{ + Clock: clock, + }, + Secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: SigningCASecretName(stackName), + Namespace: stackNamespace, + Annotations: map[string]string{ + CertificateIssuer: "dev_ns@signing-ca@10000", + CertificateNotAfterAnnotation: invalidNotAfter.Format(time.RFC3339), + CertificateNotBeforeAnnotation: invalidNotBefore.Format(time.RFC3339), + }, + }, + }, + }, + } + + err := SigningCAExpired(opts) + + e := &CertExpiredError{} + require.Error(t, err) + require.ErrorAs(t, err, &e) + require.Contains(t, err.(*CertExpiredError).Reasons, "already expired") +} + +func TestBuildSigningCASecret_Create(t *testing.T) { + opts := &Options{ + StackName: "dev", + StackNamespace: "ns", + } + + obj, err := buildSigningCASecret(opts) + require.NoError(t, err) + require.NotNil(t, obj) + require.NotNil(t, opts.Signer.RawCA) + + s := obj.(*corev1.Secret) + // Require mandatory annotations for rotation + require.Contains(t, s.Annotations, CertificateIssuer) + require.Contains(t, s.Annotations, CertificateNotAfterAnnotation) + require.Contains(t, s.Annotations, CertificateNotBeforeAnnotation) + + // Require cert-key-pair in data section + require.NotEmpty(t, s.Data[corev1.TLSCertKey]) + require.NotEmpty(t, s.Data[corev1.TLSPrivateKeyKey]) +} + +func TestBuildSigningCASecret_Rotate(t *testing.T) { + var ( + clock = time.Now + invalidNotAfter, _ = time.Parse(time.RFC3339, "") + invalidNotBefore, _ = time.Parse(time.RFC3339, "") + ) + + opts := &Options{ + StackName: "dev", + StackNamespace: "ns", + Signer: SigningCA{ + Rotation: signerRotation{ + Clock: clock, + }, + Secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dev-signing-ca", + Namespace: "ns", + Annotations: map[string]string{ + CertificateIssuer: "dev_ns@signing-ca@10000", + CertificateNotAfterAnnotation: invalidNotAfter.Format(time.RFC3339), + CertificateNotBeforeAnnotation: invalidNotBefore.Format(time.RFC3339), + }, + }, + }, + }, + } + + obj, err := buildSigningCASecret(opts) + require.NoError(t, err) + require.NotNil(t, obj) + require.NotNil(t, opts.Signer.RawCA) + + s := obj.(*corev1.Secret) + // Require mandatory annotations for rotation + require.Contains(t, s.Annotations, CertificateIssuer) + require.Contains(t, s.Annotations, CertificateNotAfterAnnotation) + require.Contains(t, s.Annotations, CertificateNotBeforeAnnotation) + + // Require cert-key-pair in data section + require.NotEmpty(t, s.Data[corev1.TLSCertKey]) + require.NotEmpty(t, s.Data[corev1.TLSPrivateKeyKey]) + + // Require rotation + require.NotEqual(t, s.Annotations[CertificateIssuer], opts.Signer.Secret.Annotations[CertificateIssuer]) + require.NotEqual(t, s.Annotations[CertificateNotAfterAnnotation], opts.Signer.Secret.Annotations[CertificateNotAfterAnnotation]) + require.NotEqual(t, s.Annotations[CertificateNotBeforeAnnotation], opts.Signer.Secret.Annotations[CertificateNotBeforeAnnotation]) + require.NotEqual(t, string(s.Data[corev1.TLSCertKey]), string(opts.Signer.Secret.Data[corev1.TLSCertKey])) + require.NotEqual(t, string(s.Data[corev1.TLSPrivateKeyKey]), string(opts.Signer.Secret.Data[corev1.TLSPrivateKeyKey])) +} diff --git a/operator/internal/certrotation/target.go b/operator/internal/certrotation/target.go new file mode 100644 index 000000000000..db49a1edd706 --- /dev/null +++ b/operator/internal/certrotation/target.go @@ -0,0 +1,126 @@ +package certrotation + +import ( + "fmt" + "time" + + "github.com/ViaQ/logerr/v2/kverrors" + "github.com/openshift/library-go/pkg/crypto" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// CertificatesExpired returns an error if any certificates expired and the list of expiry reasons. +func CertificatesExpired(opts Options) error { + if opts.Signer.Secret == nil || opts.CABundle == nil { + return nil + } + + for _, cert := range opts.Certificates { + if cert.Secret == nil { + return nil + } + } + + rawCA, err := crypto.GetCAFromBytes(opts.Signer.Secret.Data[corev1.TLSCertKey], opts.Signer.Secret.Data[corev1.TLSPrivateKeyKey]) + if err != nil { + return kverrors.Wrap(err, "failed to get signing CA from secret") + } + + caBundle := opts.CABundle.Data[CAFile] + caCerts, err := crypto.CertsFromPEM([]byte(caBundle)) + if err != nil { + return kverrors.Wrap(err, "failed to get ca bundle certificates from configmap") + } + + var reasons []string + for name, cert := range opts.Certificates { + reason := cert.Rotation.NeedNewCertificate(cert.Secret.Annotations, rawCA, caCerts, opts.Rotation.TargetCertRefresh) + if reason != "" { + reasons = append(reasons, fmt.Sprintf("%s: %s", name, reason)) + } + } + + if len(reasons) == 0 { + return nil + } + + return &CertExpiredError{Message: "certificates expired", Reasons: reasons} +} + +// buildTargetCertKeyPairSecrets returns a slice of all rotated client and serving lokistack certificates. +func buildTargetCertKeyPairSecrets(opts Options) ([]client.Object, error) { + var ( + res = make([]client.Object, 0) + ns = opts.StackNamespace + rawCA = opts.Signer.RawCA + caBundle = opts.RawCACerts + validity = opts.Rotation.TargetCertValidity + refresh = opts.Rotation.TargetCertRefresh + ) + + for name, cert := range opts.Certificates { + secret := newTargetCertificateSecret(name, ns, cert.Secret) + reason := cert.Rotation.NeedNewCertificate(secret.Annotations, rawCA, caBundle, refresh) + if len(reason) > 0 { + if err := setTargetCertKeyPairSecret(secret, validity, rawCA, cert.Rotation); err != nil { + return nil, err + } + } + + res = append(res, secret) + } + + return res, nil +} + +func newTargetCertificateSecret(name, ns string, s *corev1.Secret) *corev1.Secret { + current := s.DeepCopy() + + ss := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Type: corev1.SecretTypeTLS, + } + + if current != nil { + ss.Annotations = current.Annotations + ss.Labels = current.Labels + ss.Data = current.Data + } + + return ss +} + +// setTargetCertKeyPairSecret creates a new cert/key pair and sets them in the secret. Only one of client, serving, or signer rotation may be specified. +func setTargetCertKeyPairSecret(s *corev1.Secret, validity time.Duration, signer *crypto.CA, certCreator certificateRotation) error { + if s.Annotations == nil { + s.Annotations = map[string]string{} + } + if s.Data == nil { + s.Data = map[string][]byte{} + } + + // our annotation is based on our cert validity, so we want to make sure that we don't specify something past our signer + targetValidity := validity + remainingSignerValidity := time.Until(signer.Config.Certs[0].NotAfter) + if remainingSignerValidity < validity { + targetValidity = remainingSignerValidity + } + + certKeyPair, err := certCreator.NewCertificate(signer, targetValidity) + if err != nil { + return err + } + + s.Data[corev1.TLSCertKey], s.Data[corev1.TLSPrivateKeyKey], err = certKeyPair.GetPEMBytes() + if err != nil { + return err + } + certCreator.SetAnnotations(certKeyPair, s.Annotations) + + return nil +} diff --git a/operator/internal/certrotation/target_test.go b/operator/internal/certrotation/target_test.go new file mode 100644 index 000000000000..c705e9c50816 --- /dev/null +++ b/operator/internal/certrotation/target_test.go @@ -0,0 +1,165 @@ +package certrotation + +import ( + "crypto/x509" + "testing" + "time" + + configv1 "github.com/grafana/loki/operator/apis/config/v1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/cert" +) + +func TestCertificatesExpired(t *testing.T) { + var ( + stackName = "dev" + stackNamespce = "ns" + invalidNotAfter, _ = time.Parse(time.RFC3339, "") + invalidNotBefore, _ = time.Parse(time.RFC3339, "") + rawCA, caBytes = newTestCABundle(t, "dev-ca") + cfg = configv1.BuiltInCertManagement{ + CACertValidity: "10m", + CACertRefresh: "5m", + CertValidity: "2m", + CertRefresh: "1m", + } + ) + + certBytes, keyBytes, err := rawCA.Config.GetPEMBytes() + require.NoError(t, err) + + opts := Options{ + StackName: stackName, + StackNamespace: stackNamespce, + Signer: SigningCA{ + RawCA: rawCA, + Secret: &corev1.Secret{ + Data: map[string][]byte{ + corev1.TLSCertKey: certBytes, + corev1.TLSPrivateKeyKey: keyBytes, + }, + }, + }, + CABundle: &corev1.ConfigMap{ + Data: map[string]string{ + CAFile: string(caBytes), + }, + }, + RawCACerts: rawCA.Config.Certs, + } + err = ApplyDefaultSettings(&opts, cfg) + require.NoError(t, err) + + for _, name := range ComponentCertSecretNames(stackName) { + cert := opts.Certificates[name] + cert.Secret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: stackNamespce, + Annotations: map[string]string{ + CertificateIssuer: "dev_ns@signing-ca@10000", + CertificateNotAfterAnnotation: invalidNotAfter.Format(time.RFC3339), + CertificateNotBeforeAnnotation: invalidNotBefore.Format(time.RFC3339), + }, + }, + } + opts.Certificates[name] = cert + } + + var expired *CertExpiredError + err = CertificatesExpired(opts) + + require.Error(t, err) + require.ErrorAs(t, err, &expired) + require.Len(t, err.(*CertExpiredError).Reasons, 15) +} + +func TestBuildTargetCertKeyPairSecrets_Create(t *testing.T) { + var ( + rawCA, _ = newTestCABundle(t, "test-ca") + cfg = configv1.BuiltInCertManagement{ + CACertValidity: "10m", + CACertRefresh: "5m", + CertValidity: "2m", + CertRefresh: "1m", + } + ) + + opts := Options{ + StackName: "dev", + StackNamespace: "ns", + Signer: SigningCA{ + RawCA: rawCA, + }, + RawCACerts: rawCA.Config.Certs, + } + + err := ApplyDefaultSettings(&opts, cfg) + require.NoError(t, err) + + objs, err := buildTargetCertKeyPairSecrets(opts) + require.NoError(t, err) + require.Len(t, objs, 15) +} + +func TestBuildTargetCertKeyPairSecrets_Rotate(t *testing.T) { + var ( + rawCA, _ = newTestCABundle(t, "test-ca") + invalidNotAfter, _ = time.Parse(time.RFC3339, "") + invalidNotBefore, _ = time.Parse(time.RFC3339, "") + cfg = configv1.BuiltInCertManagement{ + CACertValidity: "10m", + CACertRefresh: "5m", + CertValidity: "2m", + CertRefresh: "1m", + } + ) + + opts := Options{ + StackName: "dev", + StackNamespace: "ns", + Signer: SigningCA{ + RawCA: rawCA, + }, + RawCACerts: rawCA.Config.Certs, + Certificates: map[string]SelfSignedCertKey{ + "dev-ingester-grpc": { + Secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dev-ingester-grpc", + Namespace: "ns", + Annotations: map[string]string{ + CertificateIssuer: "dev_ns@signing-ca@10000", + CertificateNotAfterAnnotation: invalidNotAfter.Format(time.RFC3339), + CertificateNotBeforeAnnotation: invalidNotBefore.Format(time.RFC3339), + }, + }, + }, + }, + }, + } + err := ApplyDefaultSettings(&opts, cfg) + require.NoError(t, err) + + objs, err := buildTargetCertKeyPairSecrets(opts) + require.NoError(t, err) + require.Len(t, objs, 15) + + // Check serving certificate rotation + s := objs[7].(*corev1.Secret) + ss := opts.Certificates["dev-ingester-grpc"] + + require.NotEqual(t, s.Annotations[CertificateIssuer], ss.Secret.Annotations[CertificateIssuer]) + require.NotEqual(t, s.Annotations[CertificateNotAfterAnnotation], ss.Secret.Annotations[CertificateNotAfterAnnotation]) + require.NotEqual(t, s.Annotations[CertificateNotBeforeAnnotation], ss.Secret.Annotations[CertificateNotBeforeAnnotation]) + require.NotEqual(t, s.Annotations[CertificateHostnames], ss.Secret.Annotations[CertificateHostnames]) + require.NotEqual(t, string(s.Data[corev1.TLSCertKey]), string(ss.Secret.Data[corev1.TLSCertKey])) + require.NotEqual(t, string(s.Data[corev1.TLSPrivateKeyKey]), string(ss.Secret.Data[corev1.TLSPrivateKeyKey])) + + certs, err := cert.ParseCertsPEM(s.Data[corev1.TLSCertKey]) + require.NoError(t, err) + require.Contains(t, certs[0].ExtKeyUsage, x509.ExtKeyUsageClientAuth) + require.Contains(t, certs[0].ExtKeyUsage, x509.ExtKeyUsageServerAuth) +} diff --git a/operator/internal/certrotation/var.go b/operator/internal/certrotation/var.go new file mode 100644 index 000000000000..1134ebbd135f --- /dev/null +++ b/operator/internal/certrotation/var.go @@ -0,0 +1,52 @@ +package certrotation + +import ( + "fmt" +) + +const ( + // CertificateNotBeforeAnnotation contains the certificate expiration date in RFC3339 format. + CertificateNotBeforeAnnotation = "loki.grafana.com/certificate-not-before" + // CertificateNotAfterAnnotation contains the certificate expiration date in RFC3339 format. + CertificateNotAfterAnnotation = "loki.grafana.com/certificate-not-after" + // CertificateIssuer contains the common name of the certificate that signed another certificate. + CertificateIssuer = "loki.grafana.com/certificate-issuer" + // CertificateHostnames contains the hostnames used by a signer. + CertificateHostnames = "loki.grafana.com/certificate-hostnames" +) + +const ( + // CAFile is the file name of the certificate authority file + CAFile = "service-ca.crt" +) + +// SigningCASecretName returns the lokistack signing CA secret name +func SigningCASecretName(stackName string) string { + return fmt.Sprintf("%s-signing-ca", stackName) +} + +// CABundleName returns the lokistack ca bundle configmap name +func CABundleName(stackName string) string { + return fmt.Sprintf("%s-ca-bundle", stackName) +} + +// ComponentCertSecretNames retruns a list of all loki component certificate secret names. +func ComponentCertSecretNames(stackName string) []string { + return []string{ + fmt.Sprintf("%s-gateway-client-http", stackName), + fmt.Sprintf("%s-compactor-http", stackName), + fmt.Sprintf("%s-compactor-grpc", stackName), + fmt.Sprintf("%s-distributor-http", stackName), + fmt.Sprintf("%s-distributor-grpc", stackName), + fmt.Sprintf("%s-index-gateway-http", stackName), + fmt.Sprintf("%s-index-gateway-grpc", stackName), + fmt.Sprintf("%s-ingester-http", stackName), + fmt.Sprintf("%s-ingester-grpc", stackName), + fmt.Sprintf("%s-querier-http", stackName), + fmt.Sprintf("%s-querier-grpc", stackName), + fmt.Sprintf("%s-query-frontend-http", stackName), + fmt.Sprintf("%s-query-frontend-grpc", stackName), + fmt.Sprintf("%s-ruler-http", stackName), + fmt.Sprintf("%s-ruler-grpc", stackName), + } +} diff --git a/operator/internal/handlers/internal/certificates/options.go b/operator/internal/handlers/internal/certificates/options.go new file mode 100644 index 000000000000..017f3f372d1a --- /dev/null +++ b/operator/internal/handlers/internal/certificates/options.go @@ -0,0 +1,128 @@ +package certificates + +import ( + "context" + "regexp" + + "github.com/ViaQ/logerr/v2/kverrors" + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/internal/certrotation" + "github.com/grafana/loki/operator/internal/external/k8s" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var serviceCAnnotationsRe = regexp.MustCompile(`^service.(?:alpha|beta)\.openshift\.io\/.+`) + +// GetOptions return a certrotation options struct filled with all found client and serving certificate secrets if any found. +// Return an error only if either the k8s client returns any other error except IsNotFound or if merging options fails. +func GetOptions(ctx context.Context, k k8s.Client, req ctrl.Request, mode lokiv1.ModeType) (certrotation.Options, error) { + name := certrotation.SigningCASecretName(req.Name) + ca, err := getSecret(ctx, k, name, req.Namespace) + if err != nil { + if !apierrors.IsNotFound(err) { + return certrotation.Options{}, kverrors.Wrap(err, "failed to get signing ca secret", "name", name) + } + } + + name = certrotation.CABundleName(req.Name) + bundle, err := getConfigMap(ctx, k, name, req.Namespace) + if err != nil { + if !apierrors.IsNotFound(err) { + return certrotation.Options{}, kverrors.Wrap(err, "failed to get ca bundle secret", "name", name) + } + } + configureCABundleForTenantMode(bundle, mode) + + certs, err := getCertificateOptions(ctx, k, req) + if err != nil { + return certrotation.Options{}, err + } + configureCertificatesForTenantMode(certs, mode) + + return certrotation.Options{ + StackName: req.Name, + StackNamespace: req.Namespace, + Signer: certrotation.SigningCA{ + Secret: ca, + }, + CABundle: bundle, + Certificates: certs, + }, nil +} + +func getCertificateOptions(ctx context.Context, k k8s.Client, req ctrl.Request) (certrotation.ComponentCertificates, error) { + cs := certrotation.ComponentCertSecretNames(req.Name) + certs := make(certrotation.ComponentCertificates, len(cs)) + + for _, name := range cs { + s, err := getSecret(ctx, k, name, req.Namespace) + if err != nil { + if !apierrors.IsNotFound(err) { + return nil, kverrors.Wrap(err, "failed to get secret", "name", name) + } + continue + } + + certs[name] = certrotation.SelfSignedCertKey{Secret: s} + } + + return certs, nil +} + +func getSecret(ctx context.Context, k k8s.Client, name, ns string) (*corev1.Secret, error) { + key := client.ObjectKey{Name: name, Namespace: ns} + s := &corev1.Secret{} + err := k.Get(ctx, key, s) + if err != nil { + return nil, err + } + + return s, nil +} + +func getConfigMap(ctx context.Context, k k8s.Client, name, ns string) (*corev1.ConfigMap, error) { + key := client.ObjectKey{Name: name, Namespace: ns} + s := &corev1.ConfigMap{} + err := k.Get(ctx, key, s) + if err != nil { + return nil, err + } + + return s, nil +} + +func configureCertificatesForTenantMode(certs certrotation.ComponentCertificates, mode lokiv1.ModeType) { + switch mode { + case "", lokiv1.Dynamic, lokiv1.Static: + return + case lokiv1.OpenshiftLogging, lokiv1.OpenshiftNetwork: + // Remove serviceCA annotations for existing secrets to + // enable upgrading secrets to built-in cert management + for name := range certs { + for key := range certs[name].Secret.Annotations { + if serviceCAnnotationsRe.MatchString(key) { + delete(certs[name].Secret.Annotations, key) + } + } + } + } +} + +func configureCABundleForTenantMode(cm *corev1.ConfigMap, mode lokiv1.ModeType) { + switch mode { + case "", lokiv1.Dynamic, lokiv1.Static: + return + case lokiv1.OpenshiftLogging, lokiv1.OpenshiftNetwork: + // Remove serviceCA annotations for existing ConfigMap to + // enable upgrading CABundle from built-in cert management + for key := range cm.Annotations { + if serviceCAnnotationsRe.MatchString(key) { + delete(cm.Annotations, key) + } + } + } +} diff --git a/operator/internal/handlers/internal/certificates/options_test.go b/operator/internal/handlers/internal/certificates/options_test.go new file mode 100644 index 000000000000..16603b32712c --- /dev/null +++ b/operator/internal/handlers/internal/certificates/options_test.go @@ -0,0 +1,217 @@ +package certificates + +import ( + "context" + "fmt" + "strings" + "testing" + + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/internal/external/k8s/k8sfakes" + "github.com/stretchr/testify/require" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestGetOptions_ReturnEmpty_WhenCertificatesNotExisting(t *testing.T) { + k := &k8sfakes.FakeClient{} + + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "lokistack-dev", + Namespace: "ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + opts, err := GetOptions(context.TODO(), k, req, lokiv1.Static) + require.NoError(t, err) + require.NotEmpty(t, opts) + + // Basic options always available + require.Equal(t, req.Name, opts.StackName) + require.Equal(t, req.Namespace, opts.StackNamespace) + + // Require all resource empty as per not existing + require.Nil(t, opts.Signer.Secret) + require.Nil(t, opts.CABundle) + require.Len(t, opts.Certificates, 0) +} + +func TestGetOptions_ReturnSecrets_WhenCertificatesExisting(t *testing.T) { + k := &k8sfakes.FakeClient{} + + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "lokistack-dev", + Namespace: "ns", + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + obj, ok := getManagedPKIResource(req.Name, req.Namespace, name.Name) + if !ok { + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + k.SetClientObject(object, obj) + return nil + } + + opts, err := GetOptions(context.TODO(), k, req, lokiv1.Static) + require.NoError(t, err) + require.NotEmpty(t, opts) + + // Basic options always available + require.Equal(t, req.Name, opts.StackName) + require.Equal(t, req.Namespace, opts.StackNamespace) + + // Check signingCA and CABundle populated into options + require.NotNil(t, opts.Signer.Secret) + require.NotNil(t, opts.CABundle) + + // Check client certificates populated into options + for name, cert := range opts.Certificates { + require.NotNil(t, cert.Secret, "missing name %s", name) + } +} + +func TestGetOptions_PruneServiceCAAnnotations_ForTenantMode(t *testing.T) { + k := &k8sfakes.FakeClient{} + + req := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "lokistack-dev", + Namespace: "ns", + }, + } + + pruned := []string{ + "service.alpha.openshift.io/expiry", + "service.beta.openshift.io/expiry", + "service.beta.openshift.io/originating-service-name", + "service.beta.openshift.io/originating-service-uid", + "service.beta.openshift.io/inject-cabundle", + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + obj, ok := getManagedPKIResource(req.Name, req.Namespace, name.Name) + if !ok { + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + annotations := map[string]string{} + for _, value := range pruned { + annotations[value] = "test" + } + obj.SetAnnotations(annotations) + + k.SetClientObject(object, obj) + return nil + } + + tt := []struct { + mode lokiv1.ModeType + wantPrune bool + }{ + { + mode: lokiv1.Dynamic, + }, + { + mode: lokiv1.Static, + }, + { + mode: lokiv1.OpenshiftLogging, + wantPrune: true, + }, + { + mode: lokiv1.OpenshiftNetwork, + wantPrune: true, + }, + } + + for _, tc := range tt { + tc := tc + t.Run(string(tc.mode), func(t *testing.T) { + opts, err := GetOptions(context.TODO(), k, req, tc.mode) + require.NoError(t, err) + require.NotEmpty(t, opts) + + if !tc.wantPrune { + return + } + + // Require CABundle ConfigMap annotations to be pruned + for _, annotation := range pruned { + require.NotContains(t, opts.CABundle.Annotations, annotation) + } + + // Require Certificate Secrets annotations to be pruned + for _, cert := range opts.Certificates { + for _, annotation := range pruned { + require.NotContains(t, cert.Secret.Annotations, annotation) + } + } + }) + } +} + +func getManagedPKIResource(stackName, stackNamespace, name string) (client.Object, bool) { + certNames := []string{ + "signing-ca", + "ca-bundle", + // client certificates + "gateway-client-http", + // serving certificates + "compactor-http", + "compactor-grpc", + "distributor-http", + "distributor-grpc", + "index-gateway-http", + "index-gateway-grpc", + "ingester-http", + "ingester-grpc", + "querier-http", + "querier-grpc", + "query-frontend-http", + "query-frontend-grpc", + "ruler-http", + "ruler-grpc", + } + + objsByName := map[string]client.Object{} + for _, name := range certNames { + secretName := fmt.Sprintf("%s-%s", stackName, name) + + var obj client.Object + if strings.HasSuffix(name, "ca-bundle") { + obj = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: stackNamespace, + }, + } + } else { + obj = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: stackNamespace, + }, + } + } + + objsByName[secretName] = obj + } + + obj, ok := objsByName[name] + return obj, ok +} diff --git a/operator/internal/handlers/internal/serviceaccounts/serviceaccounts.go b/operator/internal/handlers/internal/serviceaccounts/serviceaccounts.go new file mode 100644 index 000000000000..1a803c03469a --- /dev/null +++ b/operator/internal/handlers/internal/serviceaccounts/serviceaccounts.go @@ -0,0 +1,22 @@ +package serviceaccounts + +import ( + "context" + + "github.com/ViaQ/logerr/v2/kverrors" + "github.com/grafana/loki/operator/internal/external/k8s" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// GetUID return the server-side generated UID for a created serviceaccount to +// associate with a Secret of type SecretServiceaccountTokenType. Returns an error if the +// associated serviceaccount is not created or the get operation failed for any reason. +func GetUID(ctx context.Context, k k8s.Client, key client.ObjectKey) (string, error) { + sa := corev1.ServiceAccount{} + if err := k.Get(ctx, key, &sa); err != nil { + return "", kverrors.Wrap(err, "failed to fetch associated serviceaccount uid", "key", key) + } + + return string(sa.UID), nil +} diff --git a/operator/internal/handlers/lokistack_check_cert_expiry.go b/operator/internal/handlers/lokistack_check_cert_expiry.go new file mode 100644 index 000000000000..1bd2f9703a03 --- /dev/null +++ b/operator/internal/handlers/lokistack_check_cert_expiry.go @@ -0,0 +1,58 @@ +package handlers + +import ( + "context" + + "github.com/ViaQ/logerr/v2/kverrors" + "github.com/go-logr/logr" + configv1 "github.com/grafana/loki/operator/apis/config/v1" + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/internal/certrotation" + "github.com/grafana/loki/operator/internal/external/k8s" + "github.com/grafana/loki/operator/internal/handlers/internal/certificates" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + ctrl "sigs.k8s.io/controller-runtime" +) + +// CheckCertExpiry handles the case if the LokiStack managed signing CA, client and/or serving +// certificates expired. Returns true if any of those expired and an error representing the reason +// of expiry. +func CheckCertExpiry(ctx context.Context, log logr.Logger, req ctrl.Request, k k8s.Client, fg configv1.FeatureGates) error { + ll := log.WithValues("lokistack", req.String(), "event", "checkCertExpiry") + + var stack lokiv1.LokiStack + if err := k.Get(ctx, req.NamespacedName, &stack); err != nil { + if apierrors.IsNotFound(err) { + // maybe the user deleted it before we could react? Either way this isn't an issue + ll.Error(err, "could not find the requested loki stack", "name", req.String()) + return nil + } + return kverrors.Wrap(err, "failed to lookup lokistack", "name", req.String()) + } + + var mode lokiv1.ModeType + if stack.Spec.Tenants != nil { + mode = stack.Spec.Tenants.Mode + } + + opts, err := certificates.GetOptions(ctx, k, req, mode) + if err != nil { + return kverrors.Wrap(err, "failed to lookup certificates secrets", "name", req.String()) + } + + if optErr := certrotation.ApplyDefaultSettings(&opts, fg.BuiltInCertManagement); optErr != nil { + ll.Error(optErr, "failed to conform options to build settings") + return optErr + } + + if err := certrotation.SigningCAExpired(opts); err != nil { + return err + } + + if err := certrotation.CertificatesExpired(opts); err != nil { + return err + } + + return nil +} diff --git a/operator/internal/handlers/lokistack_check_cert_expiry_test.go b/operator/internal/handlers/lokistack_check_cert_expiry_test.go new file mode 100644 index 000000000000..dd94394af8c0 --- /dev/null +++ b/operator/internal/handlers/lokistack_check_cert_expiry_test.go @@ -0,0 +1,187 @@ +package handlers_test + +import ( + "context" + "errors" + "testing" + "time" + + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/internal/certrotation" + "github.com/grafana/loki/operator/internal/external/k8s/k8sfakes" + "github.com/grafana/loki/operator/internal/handlers" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestCheckCertExpiry_WhenGetReturnsNotFound_DoesNotError(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(ctx context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + k.StatusStub = func() client.StatusWriter { return sw } + + err := handlers.CheckCertExpiry(context.TODO(), logger, r, k, featureGates) + require.NoError(t, err) + + // make sure create was NOT called because the Get failed + require.Zero(t, k.CreateCallCount()) +} + +func TestCheckCertExpiry_WhenGetReturnsAnErrorOtherThanNotFound_ReturnsTheError(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + badRequestErr := apierrors.NewBadRequest("you do not belong here") + k.GetStub = func(ctx context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + return badRequestErr + } + + k.StatusStub = func() client.StatusWriter { return sw } + + err := handlers.CheckCertExpiry(context.TODO(), logger, r, k, featureGates) + + require.Equal(t, badRequestErr, errors.Unwrap(err)) + + // make sure create was NOT called because the Get failed + require.Zero(t, k.CreateCallCount()) +} + +func TestCheckCertExpiry_WhenGetOptionsReturnsSignerNotFound_DoesNotError(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + stack := lokiv1.LokiStack{ + TypeMeta: metav1.TypeMeta{ + Kind: "LokiStack", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + UID: "b23f9a38-9672-499f-8c29-15ede74d3ece", + }, + } + + k.GetStub = func(ctx context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + if name.Name == r.Name && name.Namespace == r.Namespace { + k.SetClientObject(object, &stack) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + k.StatusStub = func() client.StatusWriter { return sw } + + err := handlers.CheckCertExpiry(context.TODO(), logger, r, k, featureGates) + require.NoError(t, err) + + // make sure create was NOT called because the Get failed + require.Zero(t, k.CreateCallCount()) +} + +func TestCheckCertExpiry_WhenGetOptionsReturnsCABUndleNotFound_DoesNotError(t *testing.T) { + validNotAfter := time.Now().Add(600 * 24 * time.Hour).UTC().Format(time.RFC3339) + validNotBefore := time.Now().Add(600 * 24 * time.Hour).UTC().Format(time.RFC3339) + + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + stack := lokiv1.LokiStack{ + TypeMeta: metav1.TypeMeta{ + Kind: "LokiStack", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + UID: "b23f9a38-9672-499f-8c29-15ede74d3ece", + }, + } + + signer := corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack-signing-ca", + Namespace: "some-ns", + Labels: map[string]string{ + "app.kubernetes.io/name": "loki", + "app.kubernetes.io/provider": "openshift", + "loki.grafana.com/name": "my-stack", + + // Add custom label to fake semantic not equal + "test": "test", + }, + Annotations: map[string]string{ + certrotation.CertificateIssuer: "dev_ns@signing-ca@10000", + certrotation.CertificateNotAfterAnnotation: validNotAfter, + certrotation.CertificateNotBeforeAnnotation: validNotBefore, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "loki.grafana.com/v1", + Kind: "LokiStack", + Name: "my-stack", + UID: "b23f9a38-9672-499f-8c29-15ede74d3ece", + Controller: pointer.Bool(true), + BlockOwnerDeletion: pointer.Bool(true), + }, + }, + }, + } + + k.GetStub = func(ctx context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + if name.Name == r.Name && name.Namespace == r.Namespace { + k.SetClientObject(object, &stack) + return nil + } + if name.Name == signer.Name && name.Namespace == signer.Namespace { + k.SetClientObject(object, &signer) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + k.StatusStub = func() client.StatusWriter { return sw } + + err := handlers.CheckCertExpiry(context.TODO(), logger, r, k, featureGates) + require.NoError(t, err) + + // make sure create was NOT called because the Get failed + require.Zero(t, k.CreateCallCount()) +} diff --git a/operator/internal/handlers/lokistack_create_or_update.go b/operator/internal/handlers/lokistack_create_or_update.go index 52df276ab01e..af34ec1611a4 100644 --- a/operator/internal/handlers/lokistack_create_or_update.go +++ b/operator/internal/handlers/lokistack_create_or_update.go @@ -13,6 +13,7 @@ import ( "github.com/grafana/loki/operator/internal/handlers/internal/gateway" "github.com/grafana/loki/operator/internal/handlers/internal/openshift" "github.com/grafana/loki/operator/internal/handlers/internal/rules" + "github.com/grafana/loki/operator/internal/handlers/internal/serviceaccounts" "github.com/grafana/loki/operator/internal/handlers/internal/storage" "github.com/grafana/loki/operator/internal/handlers/internal/tlsprofile" "github.com/grafana/loki/operator/internal/manifests" @@ -29,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) // CreateOrUpdateLokiStack handles LokiStack create and update events. @@ -214,18 +216,24 @@ func CreateOrUpdateLokiStack( } + certRotationRequiredAt := "" + if stack.Annotations != nil { + certRotationRequiredAt = stack.Annotations[manifests.AnnotationCertRotationRequiredAt] + } + // Here we will translate the lokiv1.LokiStack options into manifest options opts := manifests.Options{ - Name: req.Name, - Namespace: req.Namespace, - Image: img, - GatewayImage: gwImg, - GatewayBaseDomain: baseDomain, - Stack: stack.Spec, - Gates: fg, - ObjectStorage: *objStore, - AlertingRules: alertingRules, - RecordingRules: recordingRules, + Name: req.Name, + Namespace: req.Namespace, + Image: img, + GatewayImage: gwImg, + GatewayBaseDomain: baseDomain, + Stack: stack.Spec, + Gates: fg, + ObjectStorage: *objStore, + CertRotationRequiredAt: certRotationRequiredAt, + AlertingRules: alertingRules, + RecordingRules: recordingRules, Ruler: manifests.Ruler{ Spec: rulerConfig, Secret: rulerSecret, @@ -308,8 +316,13 @@ func CreateOrUpdateLokiStack( } } + depAnnotations, err := dependentAnnotations(ctx, k, obj) + if err != nil { + return err + } + desired := obj.DeepCopyObject().(client.Object) - mutateFn := manifests.MutateFuncFor(obj, desired) + mutateFn := manifests.MutateFuncFor(obj, desired, depAnnotations) op, err := ctrl.CreateOrUpdate(ctx, k, obj, mutateFn) if err != nil { @@ -318,7 +331,13 @@ func CreateOrUpdateLokiStack( continue } - l.Info(fmt.Sprintf("Resource has been %s", op)) + msg := fmt.Sprintf("Resource has been %s", op) + switch op { + case ctrlutil.OperationResultNone: + l.V(1).Info(msg) + default: + l.Info(msg) + } } if errCount > 0 { @@ -334,6 +353,24 @@ func CreateOrUpdateLokiStack( return nil } +func dependentAnnotations(ctx context.Context, k k8s.Client, obj client.Object) (map[string]string, error) { + a := obj.GetAnnotations() + saName, ok := a[corev1.ServiceAccountNameKey] + if !ok || saName == "" { + return nil, nil + } + + key := client.ObjectKey{Name: saName, Namespace: obj.GetNamespace()} + uid, err := serviceaccounts.GetUID(ctx, k, key) + if err != nil { + return nil, err + } + + return map[string]string{ + corev1.ServiceAccountUIDKey: uid, + }, nil +} + func isNamespaceScoped(obj client.Object) bool { switch obj.(type) { case *rbacv1.ClusterRole, *rbacv1.ClusterRoleBinding: diff --git a/operator/internal/handlers/lokistack_create_or_update_test.go b/operator/internal/handlers/lokistack_create_or_update_test.go index 6ac288c30b1f..8ca7e1f1fd3b 100644 --- a/operator/internal/handlers/lokistack_create_or_update_test.go +++ b/operator/internal/handlers/lokistack_create_or_update_test.go @@ -39,8 +39,12 @@ var ( featureGates = configv1.FeatureGates{ ServiceMonitors: false, ServiceMonitorTLSEndpoints: false, - OpenShift: configv1.OpenShiftFeatureGates{ - ServingCertsService: false, + BuiltInCertManagement: configv1.BuiltInCertManagement{ + Enabled: true, + CACertValidity: "10m", + CACertRefresh: "5m", + CertValidity: "2m", + CertRefresh: "1m", }, } diff --git a/operator/internal/handlers/lokistack_rotate_certs.go b/operator/internal/handlers/lokistack_rotate_certs.go new file mode 100644 index 000000000000..12a240580025 --- /dev/null +++ b/operator/internal/handlers/lokistack_rotate_certs.go @@ -0,0 +1,110 @@ +package handlers + +import ( + "context" + "fmt" + + "github.com/ViaQ/logerr/v2/kverrors" + "github.com/go-logr/logr" + + configv1 "github.com/grafana/loki/operator/apis/config/v1" + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/internal/certrotation" + "github.com/grafana/loki/operator/internal/external/k8s" + "github.com/grafana/loki/operator/internal/handlers/internal/certificates" + "github.com/grafana/loki/operator/internal/manifests" + "github.com/grafana/loki/operator/internal/status" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// CreateOrRotateCertificates handles the LokiStack client and serving certificate creation and rotation +// including the signing CA and a ca bundle or else returns an error. It returns only a degrade-condition-worthy +// error if building the manifests fails for any reason. +func CreateOrRotateCertificates(ctx context.Context, log logr.Logger, req ctrl.Request, k k8s.Client, s *runtime.Scheme, fg configv1.FeatureGates) error { + ll := log.WithValues("lokistack", req.String(), "event", "createOrRotateCerts") + + var stack lokiv1.LokiStack + if err := k.Get(ctx, req.NamespacedName, &stack); err != nil { + if apierrors.IsNotFound(err) { + // maybe the user deleted it before we could react? Either way this isn't an issue + ll.Error(err, "could not find the requested LokiStack", "name", req.String()) + return nil + } + return kverrors.Wrap(err, "failed to lookup LokiStack", "name", req.String()) + } + + var mode lokiv1.ModeType + if stack.Spec.Tenants != nil { + mode = stack.Spec.Tenants.Mode + } + + opts, err := certificates.GetOptions(ctx, k, req, mode) + if err != nil { + return kverrors.Wrap(err, "failed to lookup certificates secrets", "name", req.String()) + } + + ll.Info("begin building certificate manifests") + + if optErr := certrotation.ApplyDefaultSettings(&opts, fg.BuiltInCertManagement); optErr != nil { + ll.Error(optErr, "failed to conform options to build settings") + return optErr + } + + objects, err := certrotation.BuildAll(opts) + if err != nil { + ll.Error(err, "failed to build certificate manifests") + return &status.DegradedError{ + Message: "Failed to rotate TLS certificates", + Reason: lokiv1.ReasonFailedCertificateRotation, + Requeue: true, + } + } + + ll.Info("certificate manifests built", "count", len(objects)) + + var errCount int32 + + for _, obj := range objects { + l := ll.WithValues( + "object_name", obj.GetName(), + "object_kind", obj.GetObjectKind(), + ) + + obj.SetNamespace(req.Namespace) + + if err := ctrl.SetControllerReference(&stack, obj, s); err != nil { + l.Error(err, "failed to set controller owner reference to resource") + errCount++ + continue + } + + desired := obj.DeepCopyObject().(client.Object) + mutateFn := manifests.MutateFuncFor(obj, desired, nil) + + op, err := ctrl.CreateOrUpdate(ctx, k, obj, mutateFn) + if err != nil { + l.Error(err, "failed to configure resource") + errCount++ + continue + } + + msg := fmt.Sprintf("Resource has been %s", op) + switch op { + case ctrlutil.OperationResultNone: + l.V(1).Info(msg) + default: + l.Info(msg) + } + } + + if errCount > 0 { + return kverrors.New("failed to create or rotate LokiStack certificates", "name", req.String()) + } + + return nil +} diff --git a/operator/internal/handlers/lokistack_rotate_certs_test.go b/operator/internal/handlers/lokistack_rotate_certs_test.go new file mode 100644 index 000000000000..9c3507cf8b66 --- /dev/null +++ b/operator/internal/handlers/lokistack_rotate_certs_test.go @@ -0,0 +1,566 @@ +package handlers_test + +import ( + "context" + "errors" + "testing" + + lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/internal/external/k8s/k8sfakes" + "github.com/grafana/loki/operator/internal/handlers" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestCreateOrRotateCertificates_WhenGetReturnsNotFound_DoesNotError(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + k.GetStub = func(ctx context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + k.StatusStub = func() client.StatusWriter { return sw } + + err := handlers.CreateOrRotateCertificates(context.TODO(), logger, r, k, scheme, featureGates) + require.NoError(t, err) + + // make sure create was NOT called because the Get failed + require.Zero(t, k.CreateCallCount()) +} + +func TestCreateOrRotateCertificates_WhenGetReturnsAnErrorOtherThanNotFound_ReturnsTheError(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + badRequestErr := apierrors.NewBadRequest("you do not belong here") + k.GetStub = func(ctx context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + return badRequestErr + } + + k.StatusStub = func() client.StatusWriter { return sw } + + err := handlers.CreateOrRotateCertificates(context.TODO(), logger, r, k, scheme, featureGates) + + require.Equal(t, badRequestErr, errors.Unwrap(err)) + + // make sure create was NOT called because the Get failed + require.Zero(t, k.CreateCallCount()) +} + +func TestCreateOrRotateCertificates_SetsNamespaceOnAllObjects(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + stack := lokiv1.LokiStack{ + TypeMeta: metav1.TypeMeta{ + Kind: "LokiStack", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + UID: "b23f9a38-9672-499f-8c29-15ede74d3ece", + }, + Spec: lokiv1.LokiStackSpec{ + Size: lokiv1.SizeOneXExtraSmall, + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-11", + }, + }, + Secret: lokiv1.ObjectStorageSecretSpec{ + Name: defaultSecret.Name, + Type: lokiv1.ObjectStorageSecretS3, + }, + }, + Tenants: &lokiv1.TenantsSpec{ + Mode: "dynamic", + Authentication: []lokiv1.AuthenticationSpec{ + { + TenantName: "test", + TenantID: "1234", + OIDC: &lokiv1.OIDCSpec{ + Secret: &lokiv1.TenantSecretSpec{ + Name: defaultGatewaySecret.Name, + }, + }, + }, + }, + Authorization: &lokiv1.AuthorizationSpec{ + OPA: &lokiv1.OPASpec{ + URL: "some-url", + }, + }, + }, + }, + } + + k.GetStub = func(_ context.Context, name types.NamespacedName, out client.Object, _ ...client.GetOption) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(out, &stack) + return nil + } + if defaultSecret.Name == name.Name { + k.SetClientObject(out, &defaultSecret) + return nil + } + if defaultGatewaySecret.Name == name.Name { + k.SetClientObject(out, &defaultGatewaySecret) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + k.CreateStub = func(_ context.Context, o client.Object, _ ...client.CreateOption) error { + assert.Equal(t, r.Namespace, o.GetNamespace()) + return nil + } + + k.StatusStub = func() client.StatusWriter { return sw } + + err := handlers.CreateOrRotateCertificates(context.TODO(), logger, r, k, scheme, featureGates) + require.NoError(t, err) + + // make sure create was called + require.NotZero(t, k.CreateCallCount()) +} + +func TestCreateOrRotateCertificates_SetsOwnerRefOnAllObjects(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + stack := lokiv1.LokiStack{ + TypeMeta: metav1.TypeMeta{ + Kind: "LokiStack", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "someStack", + Namespace: "some-ns", + UID: "b23f9a38-9672-499f-8c29-15ede74d3ece", + }, + Spec: lokiv1.LokiStackSpec{ + Size: lokiv1.SizeOneXExtraSmall, + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-11", + }, + }, + Secret: lokiv1.ObjectStorageSecretSpec{ + Name: defaultSecret.Name, + Type: lokiv1.ObjectStorageSecretS3, + }, + }, + Tenants: &lokiv1.TenantsSpec{ + Mode: "dynamic", + Authentication: []lokiv1.AuthenticationSpec{ + { + TenantName: "test", + TenantID: "1234", + OIDC: &lokiv1.OIDCSpec{ + Secret: &lokiv1.TenantSecretSpec{ + Name: defaultGatewaySecret.Name, + }, + }, + }, + }, + Authorization: &lokiv1.AuthorizationSpec{ + OPA: &lokiv1.OPASpec{ + URL: "some-url", + }, + }, + }, + }, + } + + // Create looks up the CR first, so we need to return our fake stack + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &stack) + return nil + } + if defaultSecret.Name == name.Name { + k.SetClientObject(object, &defaultSecret) + return nil + } + if defaultGatewaySecret.Name == name.Name { + k.SetClientObject(object, &defaultGatewaySecret) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something wasn't found") + } + + expected := metav1.OwnerReference{ + APIVersion: lokiv1.GroupVersion.String(), + Kind: stack.Kind, + Name: stack.Name, + UID: stack.UID, + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + } + + k.CreateStub = func(_ context.Context, o client.Object, _ ...client.CreateOption) error { + // OwnerRefs are appended so we have to find ours in the list + var ref metav1.OwnerReference + var found bool + for _, or := range o.GetOwnerReferences() { + if or.UID == stack.UID { + found = true + ref = or + break + } + } + + require.True(t, found, "expected to find a matching ownerRef, but did not") + require.EqualValues(t, expected, ref) + return nil + } + + k.StatusStub = func() client.StatusWriter { return sw } + + err := handlers.CreateOrRotateCertificates(context.TODO(), logger, r, k, scheme, featureGates) + require.NoError(t, err) + + // make sure create was called + require.NotZero(t, k.CreateCallCount()) +} + +func TestCreateOrRotateCertificates_WhenSetControllerRefInvalid_ContinueWithOtherObjects(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + stack := lokiv1.LokiStack{ + TypeMeta: metav1.TypeMeta{ + Kind: "LokiStack", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "someStack", + // Set invalid namespace here, because + // because cross-namespace controller + // references are not allowed + Namespace: "invalid-ns", + UID: "b23f9a38-9672-499f-8c29-15ede74d3ece", + }, + Spec: lokiv1.LokiStackSpec{ + Size: lokiv1.SizeOneXExtraSmall, + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-11", + }, + }, + Secret: lokiv1.ObjectStorageSecretSpec{ + Name: defaultSecret.Name, + Type: lokiv1.ObjectStorageSecretS3, + }, + }, + }, + } + + // Create looks up the CR first, so we need to return our fake stack + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &stack) + } + if defaultSecret.Name == name.Name { + k.SetClientObject(object, &defaultSecret) + } + return nil + } + + k.StatusStub = func() client.StatusWriter { return sw } + + err := handlers.CreateOrRotateCertificates(context.TODO(), logger, r, k, scheme, featureGates) + + // make sure error is returned to re-trigger reconciliation + require.Error(t, err) +} + +func TestCreateOrRotateCertificates_WhenGetReturnsNoError_UpdateObjects(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + stack := lokiv1.LokiStack{ + TypeMeta: metav1.TypeMeta{ + Kind: "LokiStack", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + UID: "b23f9a38-9672-499f-8c29-15ede74d3ece", + }, + Spec: lokiv1.LokiStackSpec{ + Size: lokiv1.SizeOneXExtraSmall, + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-11", + }, + }, + Secret: lokiv1.ObjectStorageSecretSpec{ + Name: defaultSecret.Name, + Type: lokiv1.ObjectStorageSecretS3, + }, + }, + }, + } + + secret := corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack-signing-ca", + Namespace: "some-ns", + Labels: map[string]string{ + "app.kubernetes.io/name": "loki", + "app.kubernetes.io/provider": "openshift", + "loki.grafana.com/name": "my-stack", + + // Add custom label to fake semantic not equal + "test": "test", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "loki.grafana.com/v1", + Kind: "LokiStack", + Name: "my-stack", + UID: "b23f9a38-9672-499f-8c29-15ede74d3ece", + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + }, + }, + } + + // Create looks up the CR first, so we need to return our fake stack + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &stack) + } + if defaultSecret.Name == name.Name { + k.SetClientObject(object, &defaultSecret) + } + if secret.Name == name.Name && secret.Namespace == name.Namespace { + k.SetClientObject(object, &secret) + } + return nil + } + + k.StatusStub = func() client.StatusWriter { return sw } + + err := handlers.CreateOrRotateCertificates(context.TODO(), logger, r, k, scheme, featureGates) + require.NoError(t, err) + + // make sure create not called + require.Zero(t, k.CreateCallCount()) + + // make sure update was called + require.NotZero(t, k.UpdateCallCount()) +} + +func TestCreateOrRotateCertificats_WhenCreateReturnsError_ContinueWithOtherObjects(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + stack := lokiv1.LokiStack{ + TypeMeta: metav1.TypeMeta{ + Kind: "LokiStack", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "someStack", + Namespace: "some-ns", + UID: "b23f9a38-9672-499f-8c29-15ede74d3ece", + }, + Spec: lokiv1.LokiStackSpec{ + Size: lokiv1.SizeOneXExtraSmall, + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-11", + }, + }, + Secret: lokiv1.ObjectStorageSecretSpec{ + Name: defaultSecret.Name, + Type: lokiv1.ObjectStorageSecretS3, + }, + }, + }, + } + + // GetStub looks up the CR first, so we need to return our fake stack + // return NotFound for everything else to trigger create. + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &stack) + return nil + } + if defaultSecret.Name == name.Name { + k.SetClientObject(object, &defaultSecret) + return nil + } + return apierrors.NewNotFound(schema.GroupResource{}, "something is not found") + } + + // CreateStub returns an error for each resource to trigger reconciliation a new. + k.CreateStub = func(_ context.Context, o client.Object, _ ...client.CreateOption) error { + return apierrors.NewTooManyRequestsError("too many create requests") + } + + k.StatusStub = func() client.StatusWriter { return sw } + + err := handlers.CreateOrRotateCertificates(context.TODO(), logger, r, k, scheme, featureGates) + + // make sure error is returned to re-trigger reconciliation + require.Error(t, err) +} + +func TestCreateOrRotateCertificates_WhenUpdateReturnsError_ContinueWithOtherObjects(t *testing.T) { + sw := &k8sfakes.FakeStatusWriter{} + k := &k8sfakes.FakeClient{} + r := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Name: "my-stack", + Namespace: "some-ns", + }, + } + + stack := lokiv1.LokiStack{ + TypeMeta: metav1.TypeMeta{ + Kind: "LokiStack", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack", + Namespace: "some-ns", + UID: "b23f9a38-9672-499f-8c29-15ede74d3ece", + }, + Spec: lokiv1.LokiStackSpec{ + Size: lokiv1.SizeOneXExtraSmall, + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV11, + EffectiveDate: "2020-10-11", + }, + }, + Secret: lokiv1.ObjectStorageSecretSpec{ + Name: defaultSecret.Name, + Type: lokiv1.ObjectStorageSecretS3, + }, + }, + }, + } + + secret := corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "my-stack-signing-ca", + Namespace: "some-ns", + Labels: map[string]string{ + "app.kubernetes.io/name": "loki", + "app.kubernetes.io/provider": "openshift", + "loki.grafana.com/name": "my-stack", + + // Add custom label to fake semantic not equal + "test": "test", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "loki.grafana.com/v1", + Kind: "LokiStack", + Name: "someStack", + UID: "b23f9a38-9672-499f-8c29-15ede74d3ece", + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + }, + }, + } + + // GetStub looks up the CR first, so we need to return our fake stack + // return NotFound for everything else to trigger create. + k.GetStub = func(_ context.Context, name types.NamespacedName, object client.Object, _ ...client.GetOption) error { + if r.Name == name.Name && r.Namespace == name.Namespace { + k.SetClientObject(object, &stack) + } + if defaultSecret.Name == name.Name { + k.SetClientObject(object, &defaultSecret) + } + if secret.Name == name.Name && secret.Namespace == name.Namespace { + k.SetClientObject(object, &secret) + } + return nil + } + + // CreateStub returns an error for each resource to trigger reconciliation a new. + k.UpdateStub = func(_ context.Context, o client.Object, _ ...client.UpdateOption) error { + return apierrors.NewTooManyRequestsError("too many create requests") + } + + k.StatusStub = func() client.StatusWriter { return sw } + + err := handlers.CreateOrRotateCertificates(context.TODO(), logger, r, k, scheme, featureGates) + + // make sure error is returned to re-trigger reconciliation + require.Error(t, err) +} diff --git a/operator/internal/manifests/build.go b/operator/internal/manifests/build.go index 44101d208039..68781b16be91 100644 --- a/operator/internal/manifests/build.go +++ b/operator/internal/manifests/build.go @@ -85,10 +85,6 @@ func BuildAll(opts Options) ([]client.Object, error) { res = append(res, gatewayObjects...) } - if opts.Stack.Tenants != nil { - res = configureLokiStackObjsForMode(res, opts) - } - if opts.Gates.ServiceMonitors { res = append(res, BuildServiceMonitors(opts)...) } diff --git a/operator/internal/manifests/build_test.go b/operator/internal/manifests/build_test.go index 0132616ac656..58aea0fbb4a7 100644 --- a/operator/internal/manifests/build_test.go +++ b/operator/internal/manifests/build_test.go @@ -320,20 +320,6 @@ func TestBuildAll_WithFeatureGates_OpenShift_ServingCertsService(t *testing.T) { require.NoError(t, err) svcs := []*corev1.Service{ - NewDistributorGRPCService(tst.BuildOptions), - NewDistributorHTTPService(tst.BuildOptions), - NewIngesterGRPCService(tst.BuildOptions), - NewIngesterHTTPService(tst.BuildOptions), - NewQuerierGRPCService(tst.BuildOptions), - NewQuerierHTTPService(tst.BuildOptions), - NewQueryFrontendGRPCService(tst.BuildOptions), - NewQueryFrontendHTTPService(tst.BuildOptions), - NewCompactorGRPCService(tst.BuildOptions), - NewCompactorHTTPService(tst.BuildOptions), - NewIndexGatewayGRPCService(tst.BuildOptions), - NewIndexGatewayHTTPService(tst.BuildOptions), - NewRulerHTTPService(tst.BuildOptions), - NewRulerGRPCService(tst.BuildOptions), NewGatewayHTTPService(tst.BuildOptions), } @@ -421,14 +407,14 @@ func TestBuildAll_WithFeatureGates_HTTPEncryption(t *testing.T) { expVolumeMount := corev1.VolumeMount{ Name: secretName, ReadOnly: false, - MountPath: "/var/run/tls/http", + MountPath: "/var/run/tls/http/server", } require.Contains(t, vms, expVolumeMount) require.Contains(t, args, "-server.tls-min-version=VersionTLS12") require.Contains(t, args, fmt.Sprintf("-server.tls-cipher-suites=%s", ciphers)) - require.Contains(t, args, "-server.http-tls-cert-path=/var/run/tls/http/tls.crt") - require.Contains(t, args, "-server.http-tls-key-path=/var/run/tls/http/tls.key") + require.Contains(t, args, "-server.http-tls-cert-path=/var/run/tls/http/server/tls.crt") + require.Contains(t, args, "-server.http-tls-key-path=/var/run/tls/http/server/tls.key") require.Equal(t, corev1.URISchemeHTTPS, rps) require.Equal(t, corev1.URISchemeHTTPS, lps) } @@ -500,12 +486,12 @@ func TestBuildAll_WithFeatureGates_ServiceMonitorTLSEndpoints(t *testing.T) { expVolumeMount := corev1.VolumeMount{ Name: secretName, ReadOnly: false, - MountPath: "/var/run/tls/http", + MountPath: "/var/run/tls/http/server", } require.Contains(t, vms, expVolumeMount) - require.Contains(t, args, "-server.http-tls-cert-path=/var/run/tls/http/tls.crt") - require.Contains(t, args, "-server.http-tls-key-path=/var/run/tls/http/tls.key") + require.Contains(t, args, "-server.http-tls-cert-path=/var/run/tls/http/server/tls.crt") + require.Contains(t, args, "-server.http-tls-key-path=/var/run/tls/http/server/tls.key") require.Equal(t, corev1.URISchemeHTTPS, rps) require.Equal(t, corev1.URISchemeHTTPS, lps) } @@ -658,8 +644,8 @@ func TestBuildAll_WithFeatureGates_GRPCEncryption(t *testing.T) { t.Run(name, func(t *testing.T) { secretName := secretsMap[name] args := []string{ - "-server.grpc-tls-cert-path=/var/run/tls/grpc/tls.crt", - "-server.grpc-tls-key-path=/var/run/tls/grpc/tls.key", + "-server.grpc-tls-cert-path=/var/run/tls/grpc/server/tls.crt", + "-server.grpc-tls-key-path=/var/run/tls/grpc/server/tls.key", "-server.tls-min-version=VersionTLS12", fmt.Sprintf("-server.tls-cipher-suites=%s", ciphers), } @@ -667,7 +653,7 @@ func TestBuildAll_WithFeatureGates_GRPCEncryption(t *testing.T) { vm := corev1.VolumeMount{ Name: secretName, ReadOnly: false, - MountPath: "/var/run/tls/grpc", + MountPath: "/var/run/tls/grpc/server", } v := corev1.Volume{ diff --git a/operator/internal/manifests/compactor.go b/operator/internal/manifests/compactor.go index 11874f36baed..c32a76fa6549 100644 --- a/operator/internal/manifests/compactor.go +++ b/operator/internal/manifests/compactor.go @@ -36,6 +36,13 @@ func BuildCompactor(opts Options) ([]client.Object, error) { } } + if opts.Gates.HTTPEncryption || opts.Gates.GRPCEncryption { + caBundleName := signingCABundleName(opts.Name) + if err := configureServiceCA(&statefulSet.Spec.Template.Spec, caBundleName); err != nil { + return nil, err + } + } + return []client.Object{ statefulSet, NewCompactorGRPCService(opts), @@ -121,7 +128,7 @@ func NewCompactorStatefulSet(opts Options) *appsv1.StatefulSet { } l := ComponentLabels(LabelCompactorComponent, opts.Name) - a := commonAnnotations(opts.ConfigSHA1) + a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) return &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "StatefulSet", @@ -182,9 +189,8 @@ func NewCompactorGRPCService(opts Options) *corev1.Service { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: labels, - Annotations: serviceAnnotations(serviceName, opts.Gates.OpenShift.ServingCertsService), + Name: serviceName, + Labels: labels, }, Spec: corev1.ServiceSpec{ ClusterIP: "None", @@ -212,9 +218,8 @@ func NewCompactorHTTPService(opts Options) *corev1.Service { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: labels, - Annotations: serviceAnnotations(serviceName, opts.Gates.OpenShift.ServingCertsService), + Name: serviceName, + Labels: labels, }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ @@ -232,7 +237,7 @@ func NewCompactorHTTPService(opts Options) *corev1.Service { func configureCompactorHTTPServicePKI(statefulSet *appsv1.StatefulSet, opts Options) error { serviceName := serviceNameCompactorHTTP(opts.Name) - return configureHTTPServicePKI(&statefulSet.Spec.Template.Spec, serviceName) + return configureHTTPServicePKI(&statefulSet.Spec.Template.Spec, serviceName, opts.TLSProfile.MinTLSVersion, opts.TLSCipherSuites()) } func configureCompactorGRPCServicePKI(sts *appsv1.StatefulSet, opts Options) error { diff --git a/operator/internal/manifests/compactor_test.go b/operator/internal/manifests/compactor_test.go index 81fc8c756283..1dbaf2cd6129 100644 --- a/operator/internal/manifests/compactor_test.go +++ b/operator/internal/manifests/compactor_test.go @@ -54,3 +54,23 @@ func TestNewCompactorStatefulSet_HasTemplateConfigHashAnnotation(t *testing.T) { require.Contains(t, annotations, expected) require.Equal(t, annotations[expected], "deadbeef") } + +func TestNewCompactorStatefulSet_HasTemplateCertRotationRequiredAtAnnotation(t *testing.T) { + ss := manifests.NewCompactorStatefulSet(manifests.Options{ + Name: "abcd", + Namespace: "efgh", + CertRotationRequiredAt: "deadbeef", + Stack: lokiv1.LokiStackSpec{ + StorageClassName: "standard", + Template: &lokiv1.LokiTemplateSpec{ + Compactor: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + }) + expected := "loki.grafana.com/certRotationRequiredAt" + annotations := ss.Spec.Template.Annotations + require.Contains(t, annotations, expected) + require.Equal(t, annotations[expected], "deadbeef") +} diff --git a/operator/internal/manifests/distributor.go b/operator/internal/manifests/distributor.go index 174bb6dde33b..afba9c8798e3 100644 --- a/operator/internal/manifests/distributor.go +++ b/operator/internal/manifests/distributor.go @@ -32,6 +32,13 @@ func BuildDistributor(opts Options) ([]client.Object, error) { } } + if opts.Gates.HTTPEncryption || opts.Gates.GRPCEncryption { + caBundleName := signingCABundleName(opts.Name) + if err := configureServiceCA(&deployment.Spec.Template.Spec, caBundleName); err != nil { + return nil, err + } + } + return []client.Object{ deployment, NewDistributorGRPCService(opts), @@ -117,7 +124,7 @@ func NewDistributorDeployment(opts Options) *appsv1.Deployment { } l := ComponentLabels(LabelDistributorComponent, opts.Name) - a := commonAnnotations(opts.ConfigSHA1) + a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ @@ -159,9 +166,8 @@ func NewDistributorGRPCService(opts Options) *corev1.Service { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: labels, - Annotations: serviceAnnotations(serviceName, opts.Gates.OpenShift.ServingCertsService), + Name: serviceName, + Labels: labels, }, Spec: corev1.ServiceSpec{ ClusterIP: "None", @@ -189,9 +195,8 @@ func NewDistributorHTTPService(opts Options) *corev1.Service { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: labels, - Annotations: serviceAnnotations(serviceName, opts.Gates.OpenShift.ServingCertsService), + Name: serviceName, + Labels: labels, }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ @@ -209,48 +214,23 @@ func NewDistributorHTTPService(opts Options) *corev1.Service { func configureDistributorHTTPServicePKI(deployment *appsv1.Deployment, opts Options) error { serviceName := serviceNameDistributorHTTP(opts.Name) - return configureHTTPServicePKI(&deployment.Spec.Template.Spec, serviceName) + return configureHTTPServicePKI(&deployment.Spec.Template.Spec, serviceName, opts.TLSProfile.MinTLSVersion, opts.TLSCipherSuites()) } func configureDistributorGRPCServicePKI(deployment *appsv1.Deployment, opts Options) error { - caBundleName := signingCABundleName(opts.Name) - secretVolumeSpec := corev1.PodSpec{ - Volumes: []corev1.Volume{ - { - Name: caBundleName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: caBundleName, - }, - }, - }, - }, - }, - } - secretContainerSpec := corev1.Container{ - VolumeMounts: []corev1.VolumeMount{ - { - Name: caBundleName, - ReadOnly: false, - MountPath: caBundleDir, - }, - }, Args: []string{ // Enable GRPC over TLS for ingester client "-ingester.client.tls-enabled=true", fmt.Sprintf("-ingester.client.tls-cipher-suites=%s", opts.TLSCipherSuites()), fmt.Sprintf("-ingester.client.tls-min-version=%s", opts.TLSProfile.MinTLSVersion), fmt.Sprintf("-ingester.client.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-ingester.client.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-ingester.client.tls-key-path=%s", lokiServerGRPCTLSKey()), fmt.Sprintf("-ingester.client.tls-server-name=%s", fqdn(serviceNameIngesterGRPC(opts.Name), opts.Namespace)), }, } - if err := mergo.Merge(&deployment.Spec.Template.Spec, secretVolumeSpec, mergo.WithAppendSlice); err != nil { - return kverrors.Wrap(err, "failed to merge volumes") - } - if err := mergo.Merge(&deployment.Spec.Template.Spec.Containers[0], secretContainerSpec, mergo.WithAppendSlice); err != nil { return kverrors.Wrap(err, "failed to merge container") } diff --git a/operator/internal/manifests/distributor_test.go b/operator/internal/manifests/distributor_test.go index acbdfc2cc55a..876c5406756c 100644 --- a/operator/internal/manifests/distributor_test.go +++ b/operator/internal/manifests/distributor_test.go @@ -28,7 +28,7 @@ func TestNewDistributorDeployment_SelectorMatchesLabels(t *testing.T) { } } -func TestNewDistributorDeployme_HasTemplateConfigHashAnnotation(t *testing.T) { +func TestNewDistributorDeployment_HasTemplateConfigHashAnnotation(t *testing.T) { ss := manifests.NewDistributorDeployment(manifests.Options{ Name: "abcd", Namespace: "efgh", @@ -47,3 +47,23 @@ func TestNewDistributorDeployme_HasTemplateConfigHashAnnotation(t *testing.T) { require.Contains(t, annotations, expected) require.Equal(t, annotations[expected], "deadbeef") } + +func TestNewDistributorDeployment_HasTemplateCertRotationRequiredAtAnnotation(t *testing.T) { + ss := manifests.NewDistributorDeployment(manifests.Options{ + Name: "abcd", + Namespace: "efgh", + CertRotationRequiredAt: "deadbeef", + Stack: lokiv1.LokiStackSpec{ + Template: &lokiv1.LokiTemplateSpec{ + Distributor: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + }) + + expected := "loki.grafana.com/certRotationRequiredAt" + annotations := ss.Spec.Template.Annotations + require.Contains(t, annotations, expected) + require.Equal(t, annotations[expected], "deadbeef") +} diff --git a/operator/internal/manifests/gateway.go b/operator/internal/manifests/gateway.go index 39e323d85525..7790de834ae3 100644 --- a/operator/internal/manifests/gateway.go +++ b/operator/internal/manifests/gateway.go @@ -4,6 +4,7 @@ import ( "crypto/sha1" "fmt" "path" + "regexp" "strings" "github.com/ViaQ/logerr/v2/kverrors" @@ -25,6 +26,8 @@ const ( tlsSecretVolume = "tls-secret" ) +var logsEndpointRe = regexp.MustCompile(`^--logs\.(?:read|tail|write|rules)\.endpoint=http://.+`) + // BuildGateway returns a list of k8s objects for Loki Stack Gateway func BuildGateway(opts Options) ([]client.Object, error) { cm, sha1C, err := gatewayConfigMap(opts) @@ -33,6 +36,8 @@ func BuildGateway(opts Options) ([]client.Object, error) { } dpl := NewGatewayDeployment(opts, sha1C) + sa := NewServiceAccount(opts) + saToken := NewServiceAccountTokenSecret(opts) svc := NewGatewayHTTPService(opts) ing, err := NewGatewayIngress(opts) @@ -40,29 +45,31 @@ func BuildGateway(opts Options) ([]client.Object, error) { return nil, err } - objs := []client.Object{cm, dpl, svc, ing} + objs := []client.Object{cm, dpl, sa, saToken, svc, ing} minTLSVersion := opts.TLSProfile.MinTLSVersion ciphersList := opts.TLSProfile.Ciphers ciphers := strings.Join(ciphersList, `,`) - if opts.Gates.HTTPEncryption { - serviceName := serviceNameGatewayHTTP(opts.Name) - if err := configureGatewayMetricsPKI(&dpl.Spec.Template.Spec, serviceName, minTLSVersion, ciphers); err != nil { + if opts.Stack.Rules != nil && opts.Stack.Rules.Enabled { + if err := configureGatewayRulesAPI(&dpl.Spec.Template.Spec, opts.Name, opts.Namespace); err != nil { return nil, err } } - if opts.Stack.Rules != nil && opts.Stack.Rules.Enabled { - if err := configureGatewayRulesAPI(&dpl.Spec.Template.Spec, opts.Name, opts.Namespace); err != nil { + if opts.Gates.HTTPEncryption { + serviceName := serviceNameGatewayHTTP(opts.Name) + serverCAName := gatewaySigningCABundleName(GatewayName(opts.Name)) + upstreamCAName := signingCABundleName(opts.Name) + upstreamClientName := gatewayClientSecretName(opts.Name) + if err := configureGatewayServerPKI(&dpl.Spec.Template.Spec, opts.Namespace, serviceName, serverCAName, upstreamCAName, upstreamClientName, minTLSVersion, ciphers); err != nil { return nil, err } } if opts.Stack.Tenants != nil { mode := opts.Stack.Tenants.Mode - - if err := configureGatewayDeploymentForMode(dpl, mode, opts.Gates, opts.Name, opts.Namespace, minTLSVersion, ciphers); err != nil { + if err := configureGatewayDeploymentForMode(dpl, mode, opts.Gates, minTLSVersion, ciphers); err != nil { return nil, err } @@ -81,7 +88,8 @@ func BuildGateway(opts Options) ([]client.Object, error) { // NewGatewayDeployment creates a deployment object for a lokiStack-gateway func NewGatewayDeployment(opts Options, sha1C string) *appsv1.Deployment { podSpec := corev1.PodSpec{ - Affinity: defaultAffinity(opts.Gates.DefaultNodeAffinity), + ServiceAccountName: GatewayName(opts.Name), + Affinity: defaultAffinity(opts.Gates.DefaultNodeAffinity), Volumes: []corev1.Volume{ { Name: "rbac", @@ -193,7 +201,7 @@ func NewGatewayDeployment(opts Options, sha1C string) *appsv1.Deployment { } l := ComponentLabels(LabelGatewayComponent, opts.Name) - a := commonAnnotations(sha1C) + a := commonAnnotations(sha1C, opts.CertRotationRequiredAt) return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ @@ -301,6 +309,46 @@ func NewGatewayIngress(opts Options) (*networkingv1.Ingress, error) { }, nil } +// NewServiceAccount returns a k8s object for the LokiStack Gateway +// serviceaccount. +func NewServiceAccount(opts Options) client.Object { + l := ComponentLabels(LabelGatewayComponent, opts.Name) + return &corev1.ServiceAccount{ + TypeMeta: metav1.TypeMeta{ + Kind: "ServiceAccount", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Labels: l, + Name: GatewayName(opts.Name), + Namespace: opts.Namespace, + }, + AutomountServiceAccountToken: pointer.Bool(true), + } +} + +// NewServiceAccountTokenSecret returns a k8s object for the LokiStack +// Gateway secret. This secret represents the ServiceAccountToken. +func NewServiceAccountTokenSecret(opts Options) client.Object { + l := ComponentLabels(LabelGatewayComponent, opts.Name) + + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + corev1.ServiceAccountNameKey: GatewayName(opts.Name), + }, + Labels: l, + Name: gatewayTokenSecretName(GatewayName(opts.Name)), + Namespace: opts.Namespace, + }, + Type: corev1.SecretTypeServiceAccountToken, + } +} + // gatewayConfigMap creates a configMap for rbac.yaml and tenants.yaml func gatewayConfigMap(opt Options) (*corev1.ConfigMap, string, error) { cfg := gatewayConfigOptions(opt) @@ -355,7 +403,12 @@ func gatewayConfigOptions(opt Options) gateway.Options { } } -func configureGatewayMetricsPKI(podSpec *corev1.PodSpec, serviceName, minTLSVersion, ciphers string) error { +func configureGatewayServerPKI( + podSpec *corev1.PodSpec, + namespace, serviceName, serverCAName string, + upstreamCAName, upstreamClientName string, + minTLSVersion, ciphers string, +) error { var gwIndex int for i, c := range podSpec.Containers { if c.Name == gatewayContainerName { @@ -364,63 +417,115 @@ func configureGatewayMetricsPKI(podSpec *corev1.PodSpec, serviceName, minTLSVers } } - certFile := path.Join(httpTLSDir, tlsCertFile) - keyFile := path.Join(httpTLSDir, tlsKeyFile) + gwContainer := podSpec.Containers[gwIndex].DeepCopy() + gwArgs := gwContainer.Args + gwVolumes := podSpec.Volumes - secretVolumeSpec := corev1.PodSpec{ - Volumes: []corev1.Volume{ - { - Name: tlsSecretVolume, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: serviceName, - }, + for i, a := range gwArgs { + if strings.HasPrefix(a, "--web.healthchecks.url=") { + gwArgs[i] = fmt.Sprintf("--web.healthchecks.url=https://localhost:%d", gatewayHTTPPort) + } + + if logsEndpointRe.MatchString(a) { + gwArgs[i] = strings.Replace(a, "http", "https", 1) + } + } + + serverName := fqdn(serviceName, namespace) + gwArgs = append(gwArgs, + "--tls.client-auth-type=NoClientCert", + "--tls.min-version=VersionTLS12", + fmt.Sprintf("--tls.server.cert-file=%s", gatewayServerHTTPTLSCert()), + fmt.Sprintf("--tls.server.key-file=%s", gatewayServerHTTPTLSKey()), + fmt.Sprintf("--tls.healthchecks.server-ca-file=%s", gatewaySigningCAPath()), + fmt.Sprintf("--tls.healthchecks.server-name=%s", serverName), + fmt.Sprintf("--tls.internal.server.cert-file=%s", gatewayServerHTTPTLSCert()), + fmt.Sprintf("--tls.internal.server.key-file=%s", gatewayServerHTTPTLSKey()), + fmt.Sprintf("--tls.min-version=%s", minTLSVersion), + fmt.Sprintf("--tls.cipher-suites=%s", ciphers), + fmt.Sprintf("--logs.tls.ca-file=%s", gatewayUpstreamCAPath()), + fmt.Sprintf("--logs.tls.cert-file=%s", gatewayUpstreamHTTPTLSCert()), + fmt.Sprintf("--logs.tls.key-file=%s", gatewayUpstreamHTTPTLSKey()), + ) + + gwContainer.ReadinessProbe.ProbeHandler.HTTPGet.Scheme = corev1.URISchemeHTTPS + gwContainer.LivenessProbe.ProbeHandler.HTTPGet.Scheme = corev1.URISchemeHTTPS + gwContainer.Args = gwArgs + + gwVolumes = append(gwVolumes, + corev1.Volume{ + Name: tlsSecretVolume, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceName, }, }, }, - } - secretContainerSpec := corev1.Container{ - VolumeMounts: []corev1.VolumeMount{ - { - Name: tlsSecretVolume, - ReadOnly: true, - MountPath: httpTLSDir, + corev1.Volume{ + Name: upstreamClientName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: upstreamClientName, + }, }, }, - Args: []string{ - fmt.Sprintf("--tls.internal.server.cert-file=%s", certFile), - fmt.Sprintf("--tls.internal.server.key-file=%s", keyFile), - fmt.Sprintf("--tls.min-version=%s", minTLSVersion), - fmt.Sprintf("--tls.cipher-suites=%s", ciphers), - }, - } - uriSchemeContainerSpec := corev1.Container{ - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, + corev1.Volume{ + Name: upstreamCAName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: &defaultConfigMapMode, + LocalObjectReference: corev1.LocalObjectReference{ + Name: upstreamCAName, + }, }, }, }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, + corev1.Volume{ + Name: serverCAName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: &defaultConfigMapMode, + LocalObjectReference: corev1.LocalObjectReference{ + Name: serverCAName, + }, }, }, }, - } - - if err := mergo.Merge(podSpec, secretVolumeSpec, mergo.WithAppendSlice); err != nil { - return kverrors.Wrap(err, "failed to merge volumes") - } + ) + + gwContainer.VolumeMounts = append( + gwContainer.VolumeMounts, + corev1.VolumeMount{ + Name: tlsSecretVolume, + ReadOnly: true, + MountPath: gatewayServerHTTPTLSDir(), + }, + corev1.VolumeMount{ + Name: upstreamClientName, + ReadOnly: true, + MountPath: gatewayUpstreamHTTPTLSDir(), + }, + corev1.VolumeMount{ + Name: upstreamCAName, + ReadOnly: true, + MountPath: gatewayUpstreamCADir(), + }, + corev1.VolumeMount{ + Name: serverCAName, + ReadOnly: true, + MountPath: gatewaySigningCADir(), + }, + ) - if err := mergo.Merge(&podSpec.Containers[gwIndex], secretContainerSpec, mergo.WithAppendSlice); err != nil { - return kverrors.Wrap(err, "failed to merge container") + p := corev1.PodSpec{ + Containers: []corev1.Container{ + *gwContainer, + }, + Volumes: gwVolumes, } - if err := mergo.Merge(&podSpec.Containers[gwIndex], uriSchemeContainerSpec, mergo.WithOverride); err != nil { - return kverrors.Wrap(err, "failed to merge container") + if err := mergo.Merge(podSpec, p, mergo.WithOverride); err != nil { + return kverrors.Wrap(err, "failed to merge server pki into container spec ") } return nil diff --git a/operator/internal/manifests/gateway_tenants.go b/operator/internal/manifests/gateway_tenants.go index 6adcc11d5f67..9598dba13f81 100644 --- a/operator/internal/manifests/gateway_tenants.go +++ b/operator/internal/manifests/gateway_tenants.go @@ -59,38 +59,13 @@ func ApplyGatewayDefaultOptions(opts *Options) error { return nil } -func configureGatewayDeploymentForMode( - d *appsv1.Deployment, mode lokiv1.ModeType, - fg configv1.FeatureGates, stackName, stackNs string, - minTLSVersion string, ciphers string, -) error { +func configureGatewayDeploymentForMode(d *appsv1.Deployment, mode lokiv1.ModeType, fg configv1.FeatureGates, minTLSVersion string, ciphers string) error { switch mode { case lokiv1.Static, lokiv1.Dynamic: return nil // nothing to configure case lokiv1.OpenshiftLogging, lokiv1.OpenshiftNetwork: - caBundleName := signingCABundleName(stackName) - serviceName := serviceNameGatewayHTTP(stackName) - secretName := signingServiceSecretName(serviceName) - serverName := fqdn(serviceName, stackNs) - return openshift.ConfigureGatewayDeployment( - d, - mode, - gatewayContainerName, - tlsSecretVolume, - httpTLSDir, - tlsCertFile, - tlsKeyFile, - caBundleName, - caBundleDir, - caFile, - fg.HTTPEncryption, - fg.OpenShift.ServingCertsService, - secretName, - serverName, - gatewayHTTPPort, - minTLSVersion, - ciphers, - ) + tlsDir := gatewayServerHTTPTLSDir() + return openshift.ConfigureGatewayDeployment(d, mode, tlsSecretVolume, tlsDir, minTLSVersion, ciphers, fg.HTTPEncryption) } return nil @@ -107,23 +82,25 @@ func configureGatewayServiceForMode(s *corev1.ServiceSpec, mode lokiv1.ModeType) return nil } -func configureLokiStackObjsForMode(objs []client.Object, opts Options) []client.Object { - switch opts.Stack.Tenants.Mode { - case lokiv1.Static, lokiv1.Dynamic: - // nothing to configure - case lokiv1.OpenshiftLogging, lokiv1.OpenshiftNetwork: - openShiftObjs := openshift.BuildLokiStackObjects(opts.OpenShiftOptions) - objs = append(objs, openShiftObjs...) - } - - return objs -} - func configureGatewayObjsForMode(objs []client.Object, opts Options) []client.Object { switch opts.Stack.Tenants.Mode { case lokiv1.Static, lokiv1.Dynamic: // nothing to configure case lokiv1.OpenshiftLogging, lokiv1.OpenshiftNetwork: + for _, o := range objs { + switch sa := o.(type) { + case *corev1.ServiceAccount: + if sa.Annotations == nil { + sa.Annotations = map[string]string{} + } + + a := openshift.ServiceAccountAnnotations(opts.OpenShiftOptions) + for key, value := range a { + sa.Annotations[key] = value + } + } + } + openShiftObjs := openshift.BuildGatewayObjects(opts.OpenShiftOptions) var cObjs []client.Object @@ -145,12 +122,12 @@ func configureGatewayObjsForMode(objs []client.Object, opts Options) []client.Ob return objs } -func configureGatewayServiceMonitorForMode(sm *monitoringv1.ServiceMonitor, mode lokiv1.ModeType, fg configv1.FeatureGates) error { - switch mode { +func configureGatewayServiceMonitorForMode(sm *monitoringv1.ServiceMonitor, opts Options) error { + switch opts.Stack.Tenants.Mode { case lokiv1.Static, lokiv1.Dynamic: return nil // nothing to configure case lokiv1.OpenshiftLogging, lokiv1.OpenshiftNetwork: - return openshift.ConfigureGatewayServiceMonitor(sm, fg.ServiceMonitorTLSEndpoints) + return openshift.ConfigureGatewayServiceMonitor(sm, opts.Gates.ServiceMonitorTLSEndpoints) } return nil diff --git a/operator/internal/manifests/gateway_tenants_test.go b/operator/internal/manifests/gateway_tenants_test.go index c50101f3bd6b..da6c3813c50d 100644 --- a/operator/internal/manifests/gateway_tenants_test.go +++ b/operator/internal/manifests/gateway_tenants_test.go @@ -1,7 +1,7 @@ package manifests import ( - "fmt" + "path" "testing" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -280,26 +280,6 @@ func TestConfigureDeploymentForMode(t *testing.T) { Containers: []corev1.Container{ { Name: gatewayContainerName, - Args: []string{ - "--logs.read.endpoint=http://example.com", - "--logs.tail.endpoint=http://example.com", - "--logs.write.endpoint=http://example.com", - fmt.Sprintf("--web.healthchecks.url=http://localhost:%d", gatewayHTTPPort), - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTP, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTP, - }, - }, - }, }, }, }, @@ -316,40 +296,6 @@ func TestConfigureDeploymentForMode(t *testing.T) { Containers: []corev1.Container{ { Name: gatewayContainerName, - Args: []string{ - "--logs.read.endpoint=http://example.com", - "--logs.tail.endpoint=http://example.com", - "--logs.write.endpoint=http://example.com", - fmt.Sprintf("--web.healthchecks.url=https://localhost:%d", gatewayHTTPPort), - "--tls.client-auth-type=NoClientCert", - "--tls.server.cert-file=/var/run/tls/http/tls.crt", - "--tls.server.key-file=/var/run/tls/http/tls.key", - "--tls.healthchecks.server-ca-file=/var/run/ca/service-ca.crt", - fmt.Sprintf("--tls.healthchecks.server-name=%s", "test-gateway-http.test-ns.svc.cluster.local"), - "--tls.min-version=min-version", - "--tls.cipher-suites=cipher1,cipher2", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: tlsSecretVolume, - ReadOnly: true, - MountPath: httpTLSDir, - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, }, { Name: "opa", @@ -405,23 +351,13 @@ func TestConfigureDeploymentForMode(t *testing.T) { }, }, }, - Volumes: []corev1.Volume{ - { - Name: tlsSecretVolume, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "test-gateway-http-tls", - }, - }, - }, - }, }, }, }, }, }, { - desc: "openshift-logging mode with-tls-service-monitor-config", + desc: "openshift-logging mode with http encryption", mode: lokiv1.OpenshiftLogging, stackName: "test", stackNs: "test-ns", @@ -431,6 +367,7 @@ func TestConfigureDeploymentForMode(t *testing.T) { }, dpl: &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ + Name: "test-gateway", Namespace: "test-ns", }, Spec: appsv1.DeploymentSpec{ @@ -439,229 +376,6 @@ func TestConfigureDeploymentForMode(t *testing.T) { Containers: []corev1.Container{ { Name: gatewayContainerName, - Args: []string{ - "--logs.read.endpoint=http://example.com", - "--logs.tail.endpoint=http://example.com", - "--logs.write.endpoint=http://example.com", - fmt.Sprintf("--web.healthchecks.url=http://localhost:%d", gatewayHTTPPort), - "--tls.min-version=min-version", - "--tls.cipher-suites=cipher1,cipher2", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: tlsSecretVolume, - ReadOnly: true, - MountPath: httpTLSDir, - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTP, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTP, - }, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: tlsSecretVolume, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "test-gateway-http-tls", - }, - }, - }, - }, - }, - }, - }, - }, - want: &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test-ns", - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: gatewayContainerName, - Args: []string{ - "--logs.read.endpoint=http://example.com", - "--logs.tail.endpoint=http://example.com", - "--logs.write.endpoint=http://example.com", - fmt.Sprintf("--web.healthchecks.url=https://localhost:%d", gatewayHTTPPort), - "--tls.min-version=min-version", - "--tls.cipher-suites=cipher1,cipher2", - "--tls.client-auth-type=NoClientCert", - "--tls.server.cert-file=/var/run/tls/http/tls.crt", - "--tls.server.key-file=/var/run/tls/http/tls.key", - "--tls.healthchecks.server-ca-file=/var/run/ca/service-ca.crt", - fmt.Sprintf("--tls.healthchecks.server-name=%s", "test-gateway-http.test-ns.svc.cluster.local"), - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: tlsSecretVolume, - ReadOnly: true, - MountPath: httpTLSDir, - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, - }, - { - Name: "opa", - Image: "quay.io/observatorium/opa-openshift:latest", - Args: []string{ - "--log.level=warn", - "--opa.skip-tenants=audit,infrastructure", - "--opa.admin-groups=system:cluster-admins,cluster-admin,dedicated-admin", - "--web.listen=:8082", - "--web.internal.listen=:8083", - "--web.healthchecks.url=http://localhost:8082", - "--opa.package=lokistack", - "--opa.matcher=kubernetes_namespace_name", - "--tls.internal.server.cert-file=/var/run/tls/http/tls.crt", - "--tls.internal.server.key-file=/var/run/tls/http/tls.key", - "--tls.min-version=min-version", - "--tls.cipher-suites=cipher1,cipher2", - `--openshift.mappings=application=loki.grafana.com`, - `--openshift.mappings=infrastructure=loki.grafana.com`, - `--openshift.mappings=audit=loki.grafana.com`, - }, - Ports: []corev1.ContainerPort{ - { - Name: openshift.GatewayOPAHTTPPortName, - ContainerPort: openshift.GatewayOPAHTTPPort, - Protocol: corev1.ProtocolTCP, - }, - { - Name: openshift.GatewayOPAInternalPortName, - ContainerPort: openshift.GatewayOPAInternalPort, - Protocol: corev1.ProtocolTCP, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/live", - Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), - Scheme: corev1.URISchemeHTTPS, - }, - }, - TimeoutSeconds: 2, - PeriodSeconds: 30, - FailureThreshold: 10, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/ready", - Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), - Scheme: corev1.URISchemeHTTPS, - }, - }, - TimeoutSeconds: 1, - PeriodSeconds: 5, - FailureThreshold: 12, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: tlsSecretVolume, - ReadOnly: true, - MountPath: httpTLSDir, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: tlsSecretVolume, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "test-gateway-http-tls", - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - desc: "openshift-logging mode with-cert-signing-service", - mode: lokiv1.OpenshiftLogging, - stackName: "test", - stackNs: "test-ns", - featureGates: configv1.FeatureGates{ - HTTPEncryption: true, - ServiceMonitorTLSEndpoints: true, - OpenShift: configv1.OpenShiftFeatureGates{ - ServingCertsService: true, - }, - }, - dpl: &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gateway", - Namespace: "test-ns", - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: gatewayContainerName, - Args: []string{ - "--other.args=foo-bar", - "--logs.read.endpoint=http://example.com", - "--logs.tail.endpoint=http://example.com", - "--logs.write.endpoint=http://example.com", - fmt.Sprintf("--web.healthchecks.url=http://localhost:%d", gatewayHTTPPort), - "--tls.min-version=min-version", - "--tls.cipher-suites=cipher1,cipher2", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "tls-secret", - ReadOnly: true, - MountPath: "/var/run/tls/http", - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTP, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTP, - }, - }, - }, }, }, Volumes: []corev1.Volume{ @@ -672,232 +386,19 @@ func TestConfigureDeploymentForMode(t *testing.T) { }, }, }, - }, - want: &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "gateway", - Namespace: "test-ns", - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - ServiceAccountName: "gateway", - Containers: []corev1.Container{ - { - Name: gatewayContainerName, - Args: []string{ - "--other.args=foo-bar", - "--logs.read.endpoint=https://example.com", - "--logs.tail.endpoint=https://example.com", - "--logs.write.endpoint=https://example.com", - fmt.Sprintf("--web.healthchecks.url=https://localhost:%d", gatewayHTTPPort), - "--tls.min-version=min-version", - "--tls.cipher-suites=cipher1,cipher2", - "--logs.tls.ca-file=/var/run/ca/service-ca.crt", - "--tls.client-auth-type=NoClientCert", - "--tls.server.cert-file=/var/run/tls/http/tls.crt", - "--tls.server.key-file=/var/run/tls/http/tls.key", - "--tls.healthchecks.server-ca-file=/var/run/ca/service-ca.crt", - fmt.Sprintf("--tls.healthchecks.server-name=%s", "test-gateway-http.test-ns.svc.cluster.local"), - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "tls-secret", - ReadOnly: true, - MountPath: "/var/run/tls/http", - }, - { - Name: "test-ca-bundle", - ReadOnly: true, - MountPath: "/var/run/ca", - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, - }, - { - Name: "opa", - Image: "quay.io/observatorium/opa-openshift:latest", - Args: []string{ - "--log.level=warn", - "--opa.skip-tenants=audit,infrastructure", - "--opa.admin-groups=system:cluster-admins,cluster-admin,dedicated-admin", - "--web.listen=:8082", - "--web.internal.listen=:8083", - "--web.healthchecks.url=http://localhost:8082", - "--opa.package=lokistack", - "--opa.matcher=kubernetes_namespace_name", - "--tls.internal.server.cert-file=/var/run/tls/http/tls.crt", - "--tls.internal.server.key-file=/var/run/tls/http/tls.key", - "--tls.min-version=min-version", - "--tls.cipher-suites=cipher1,cipher2", - `--openshift.mappings=application=loki.grafana.com`, - `--openshift.mappings=infrastructure=loki.grafana.com`, - `--openshift.mappings=audit=loki.grafana.com`, - }, - Ports: []corev1.ContainerPort{ - { - Name: openshift.GatewayOPAHTTPPortName, - ContainerPort: openshift.GatewayOPAHTTPPort, - Protocol: corev1.ProtocolTCP, - }, - { - Name: openshift.GatewayOPAInternalPortName, - ContainerPort: openshift.GatewayOPAInternalPort, - Protocol: corev1.ProtocolTCP, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/live", - Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), - Scheme: corev1.URISchemeHTTPS, - }, - }, - TimeoutSeconds: 2, - PeriodSeconds: 30, - FailureThreshold: 10, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/ready", - Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), - Scheme: corev1.URISchemeHTTPS, - }, - }, - TimeoutSeconds: 1, - PeriodSeconds: 5, - FailureThreshold: 12, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: tlsSecretVolume, - ReadOnly: true, - MountPath: httpTLSDir, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "tls-secret-volume", - }, - { - Name: "test-ca-bundle", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - DefaultMode: &defaultConfigMapMode, - LocalObjectReference: corev1.LocalObjectReference{ - Name: "test-ca-bundle", - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - desc: "openshift-network mode", - mode: lokiv1.OpenshiftNetwork, - stackName: "test", - stackNs: "test-ns", - dpl: &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test-ns", - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: gatewayContainerName, - Args: []string{ - "--logs.read.endpoint=http://example.com", - "--logs.tail.endpoint=http://example.com", - "--logs.write.endpoint=http://example.com", - fmt.Sprintf("--web.healthchecks.url=http://localhost:%d", gatewayHTTPPort), - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTP, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTP, - }, - }, - }, - }, - }, - }, - }, - }, - }, - want: &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "test-ns", - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: gatewayContainerName, - Args: []string{ - "--logs.read.endpoint=http://example.com", - "--logs.tail.endpoint=http://example.com", - "--logs.write.endpoint=http://example.com", - fmt.Sprintf("--web.healthchecks.url=https://localhost:%d", gatewayHTTPPort), - "--tls.client-auth-type=NoClientCert", - "--tls.server.cert-file=/var/run/tls/http/tls.crt", - "--tls.server.key-file=/var/run/tls/http/tls.key", - "--tls.healthchecks.server-ca-file=/var/run/ca/service-ca.crt", - fmt.Sprintf("--tls.healthchecks.server-name=%s", "test-gateway-http.test-ns.svc.cluster.local"), - "--tls.min-version=min-version", - "--tls.cipher-suites=cipher1,cipher2", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: tlsSecretVolume, - ReadOnly: true, - MountPath: httpTLSDir, - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, + }, + want: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-gateway", + Namespace: "test-ns", + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + ServiceAccountName: "test-gateway", + Containers: []corev1.Container{ + { + Name: gatewayContainerName, }, { Name: "opa", @@ -910,7 +411,14 @@ func TestConfigureDeploymentForMode(t *testing.T) { "--web.internal.listen=:8083", "--web.healthchecks.url=http://localhost:8082", "--opa.package=lokistack", - `--openshift.mappings=network=loki.grafana.com`, + "--opa.matcher=kubernetes_namespace_name", + "--tls.internal.server.cert-file=/var/run/tls/http/server/tls.crt", + "--tls.internal.server.key-file=/var/run/tls/http/server/tls.key", + "--tls.min-version=min-version", + "--tls.cipher-suites=cipher1,cipher2", + `--openshift.mappings=application=loki.grafana.com`, + `--openshift.mappings=infrastructure=loki.grafana.com`, + `--openshift.mappings=audit=loki.grafana.com`, }, Ports: []corev1.ContainerPort{ { @@ -929,7 +437,7 @@ func TestConfigureDeploymentForMode(t *testing.T) { HTTPGet: &corev1.HTTPGetAction{ Path: "/live", Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), - Scheme: corev1.URISchemeHTTP, + Scheme: corev1.URISchemeHTTPS, }, }, TimeoutSeconds: 2, @@ -941,23 +449,25 @@ func TestConfigureDeploymentForMode(t *testing.T) { HTTPGet: &corev1.HTTPGetAction{ Path: "/ready", Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), - Scheme: corev1.URISchemeHTTP, + Scheme: corev1.URISchemeHTTPS, }, }, TimeoutSeconds: 1, PeriodSeconds: 5, FailureThreshold: 12, }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: tlsSecretVolume, + ReadOnly: true, + MountPath: gatewayServerHTTPTLSDir(), + }, + }, }, }, Volumes: []corev1.Volume{ { - Name: tlsSecretVolume, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "test-gateway-http-tls", - }, - }, + Name: "tls-secret-volume", }, }, }, @@ -966,14 +476,10 @@ func TestConfigureDeploymentForMode(t *testing.T) { }, }, { - desc: "openshift-network mode with-tls-service-monitor-config", + desc: "openshift-network mode", mode: lokiv1.OpenshiftNetwork, stackName: "test", stackNs: "test-ns", - featureGates: configv1.FeatureGates{ - HTTPEncryption: true, - ServiceMonitorTLSEndpoints: true, - }, dpl: &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Namespace: "test-ns", @@ -984,45 +490,11 @@ func TestConfigureDeploymentForMode(t *testing.T) { Containers: []corev1.Container{ { Name: gatewayContainerName, - Args: []string{ - "--logs.read.endpoint=http://example.com", - "--logs.tail.endpoint=http://example.com", - "--logs.write.endpoint=http://example.com", - fmt.Sprintf("--web.healthchecks.url=http://localhost:%d", gatewayHTTPPort), - "--tls.min-version=min-version", - "--tls.cipher-suites=cipher1,cipher2", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: tlsSecretVolume, - ReadOnly: true, - MountPath: httpTLSDir, - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTP, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTP, - }, - }, - }, }, }, Volumes: []corev1.Volume{ { - Name: tlsSecretVolume, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "test-gateway-http-tls", - }, - }, + Name: "tls-secret-volume", }, }, }, @@ -1039,40 +511,6 @@ func TestConfigureDeploymentForMode(t *testing.T) { Containers: []corev1.Container{ { Name: gatewayContainerName, - Args: []string{ - "--logs.read.endpoint=http://example.com", - "--logs.tail.endpoint=http://example.com", - "--logs.write.endpoint=http://example.com", - fmt.Sprintf("--web.healthchecks.url=https://localhost:%d", gatewayHTTPPort), - "--tls.min-version=min-version", - "--tls.cipher-suites=cipher1,cipher2", - "--tls.client-auth-type=NoClientCert", - "--tls.server.cert-file=/var/run/tls/http/tls.crt", - "--tls.server.key-file=/var/run/tls/http/tls.key", - "--tls.healthchecks.server-ca-file=/var/run/ca/service-ca.crt", - fmt.Sprintf("--tls.healthchecks.server-name=%s", "test-gateway-http.test-ns.svc.cluster.local"), - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: tlsSecretVolume, - ReadOnly: true, - MountPath: httpTLSDir, - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, }, { Name: "opa", @@ -1085,10 +523,6 @@ func TestConfigureDeploymentForMode(t *testing.T) { "--web.internal.listen=:8083", "--web.healthchecks.url=http://localhost:8082", "--opa.package=lokistack", - "--tls.internal.server.cert-file=/var/run/tls/http/tls.crt", - "--tls.internal.server.key-file=/var/run/tls/http/tls.key", - "--tls.min-version=min-version", - "--tls.cipher-suites=cipher1,cipher2", `--openshift.mappings=network=loki.grafana.com`, }, Ports: []corev1.ContainerPort{ @@ -1108,7 +542,7 @@ func TestConfigureDeploymentForMode(t *testing.T) { HTTPGet: &corev1.HTTPGetAction{ Path: "/live", Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), - Scheme: corev1.URISchemeHTTPS, + Scheme: corev1.URISchemeHTTP, }, }, TimeoutSeconds: 2, @@ -1120,30 +554,18 @@ func TestConfigureDeploymentForMode(t *testing.T) { HTTPGet: &corev1.HTTPGetAction{ Path: "/ready", Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), - Scheme: corev1.URISchemeHTTPS, + Scheme: corev1.URISchemeHTTP, }, }, TimeoutSeconds: 1, PeriodSeconds: 5, FailureThreshold: 12, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: tlsSecretVolume, - ReadOnly: true, - MountPath: httpTLSDir, - }, - }, }, }, Volumes: []corev1.Volume{ { - Name: tlsSecretVolume, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "test-gateway-http-tls", - }, - }, + Name: "tls-secret-volume", }, }, }, @@ -1152,20 +574,16 @@ func TestConfigureDeploymentForMode(t *testing.T) { }, }, { - desc: "openshift-network mode with-cert-signing-service", + desc: "openshift-network mode with http encryption", mode: lokiv1.OpenshiftNetwork, stackName: "test", stackNs: "test-ns", featureGates: configv1.FeatureGates{ HTTPEncryption: true, ServiceMonitorTLSEndpoints: true, - OpenShift: configv1.OpenShiftFeatureGates{ - ServingCertsService: true, - }, }, dpl: &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "gateway", Namespace: "test-ns", }, Spec: appsv1.DeploymentSpec{ @@ -1174,36 +592,6 @@ func TestConfigureDeploymentForMode(t *testing.T) { Containers: []corev1.Container{ { Name: gatewayContainerName, - Args: []string{ - "--other.args=foo-bar", - "--logs.read.endpoint=http://example.com", - "--logs.tail.endpoint=http://example.com", - "--logs.write.endpoint=http://example.com", - fmt.Sprintf("--web.healthchecks.url=http://localhost:%d", gatewayHTTPPort), - "--tls.min-version=min-version", - "--tls.cipher-suites=cipher1,cipher2", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "tls-secret", - ReadOnly: true, - MountPath: "/var/run/tls/http", - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTP, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTP, - }, - }, - }, }, }, Volumes: []corev1.Volume{ @@ -1217,57 +605,14 @@ func TestConfigureDeploymentForMode(t *testing.T) { }, want: &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "gateway", Namespace: "test-ns", }, Spec: appsv1.DeploymentSpec{ Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ - ServiceAccountName: "gateway", Containers: []corev1.Container{ { Name: gatewayContainerName, - Args: []string{ - "--other.args=foo-bar", - "--logs.read.endpoint=https://example.com", - "--logs.tail.endpoint=https://example.com", - "--logs.write.endpoint=https://example.com", - fmt.Sprintf("--web.healthchecks.url=https://localhost:%d", gatewayHTTPPort), - "--tls.min-version=min-version", - "--tls.cipher-suites=cipher1,cipher2", - "--logs.tls.ca-file=/var/run/ca/service-ca.crt", - "--tls.client-auth-type=NoClientCert", - "--tls.server.cert-file=/var/run/tls/http/tls.crt", - "--tls.server.key-file=/var/run/tls/http/tls.key", - "--tls.healthchecks.server-ca-file=/var/run/ca/service-ca.crt", - fmt.Sprintf("--tls.healthchecks.server-name=%s", "test-gateway-http.test-ns.svc.cluster.local"), - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "tls-secret", - ReadOnly: true, - MountPath: "/var/run/tls/http", - }, - { - Name: "test-ca-bundle", - ReadOnly: true, - MountPath: "/var/run/ca", - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, }, { Name: "opa", @@ -1280,8 +625,8 @@ func TestConfigureDeploymentForMode(t *testing.T) { "--web.internal.listen=:8083", "--web.healthchecks.url=http://localhost:8082", "--opa.package=lokistack", - "--tls.internal.server.cert-file=/var/run/tls/http/tls.crt", - "--tls.internal.server.key-file=/var/run/tls/http/tls.key", + "--tls.internal.server.cert-file=/var/run/tls/http/server/tls.crt", + "--tls.internal.server.key-file=/var/run/tls/http/server/tls.key", "--tls.min-version=min-version", "--tls.cipher-suites=cipher1,cipher2", `--openshift.mappings=network=loki.grafana.com`, @@ -1326,7 +671,7 @@ func TestConfigureDeploymentForMode(t *testing.T) { { Name: tlsSecretVolume, ReadOnly: true, - MountPath: httpTLSDir, + MountPath: path.Join(httpTLSDir, "server"), }, }, }, @@ -1335,17 +680,6 @@ func TestConfigureDeploymentForMode(t *testing.T) { { Name: "tls-secret-volume", }, - { - Name: "test-ca-bundle", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - DefaultMode: &defaultConfigMapMode, - LocalObjectReference: corev1.LocalObjectReference{ - Name: "test-ca-bundle", - }, - }, - }, - }, }, }, }, @@ -1358,7 +692,7 @@ func TestConfigureDeploymentForMode(t *testing.T) { tc := tc t.Run(tc.desc, func(t *testing.T) { t.Parallel() - err := configureGatewayDeploymentForMode(tc.dpl, tc.mode, tc.featureGates, "test", "test-ns", "min-version", "cipher1,cipher2") + err := configureGatewayDeploymentForMode(tc.dpl, tc.mode, tc.featureGates, "min-version", "cipher1,cipher2") require.NoError(t, err) require.Equal(t, tc.want, tc.dpl) }) @@ -1427,6 +761,7 @@ func TestConfigureServiceForMode(t *testing.T) { func TestConfigureServiceMonitorForMode(t *testing.T) { type tt struct { desc string + opts Options mode lokiv1.ModeType featureGates configv1.FeatureGates sm *monitoringv1.ServiceMonitor @@ -1436,20 +771,38 @@ func TestConfigureServiceMonitorForMode(t *testing.T) { tc := []tt{ { desc: "static mode", - mode: lokiv1.Static, + opts: Options{ + Stack: lokiv1.LokiStackSpec{ + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.Static, + }, + }, + }, sm: &monitoringv1.ServiceMonitor{}, want: &monitoringv1.ServiceMonitor{}, }, { desc: "dynamic mode", - mode: lokiv1.Dynamic, + opts: Options{ + Stack: lokiv1.LokiStackSpec{ + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.Dynamic, + }, + }, + }, sm: &monitoringv1.ServiceMonitor{}, want: &monitoringv1.ServiceMonitor{}, }, { desc: "openshift-logging mode", - mode: lokiv1.OpenshiftLogging, - sm: &monitoringv1.ServiceMonitor{}, + opts: Options{ + Stack: lokiv1.LokiStackSpec{ + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.OpenshiftLogging, + }, + }, + }, + sm: &monitoringv1.ServiceMonitor{}, want: &monitoringv1.ServiceMonitor{ Spec: monitoringv1.ServiceMonitorSpec{ Endpoints: []monitoringv1.Endpoint{ @@ -1464,8 +817,14 @@ func TestConfigureServiceMonitorForMode(t *testing.T) { }, { desc: "openshift-network mode", - mode: lokiv1.OpenshiftNetwork, - sm: &monitoringv1.ServiceMonitor{}, + opts: Options{ + Stack: lokiv1.LokiStackSpec{ + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.OpenshiftNetwork, + }, + }, + }, + sm: &monitoringv1.ServiceMonitor{}, want: &monitoringv1.ServiceMonitor{ Spec: monitoringv1.ServiceMonitorSpec{ Endpoints: []monitoringv1.Endpoint{ @@ -1480,10 +839,89 @@ func TestConfigureServiceMonitorForMode(t *testing.T) { }, { desc: "openshift-logging mode with-tls-service-monitor-config", - mode: lokiv1.OpenshiftLogging, - featureGates: configv1.FeatureGates{ - HTTPEncryption: true, - ServiceMonitorTLSEndpoints: true, + opts: Options{ + Name: "abcd", + Namespace: "ns", + Stack: lokiv1.LokiStackSpec{ + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.OpenshiftLogging, + }, + }, + Gates: configv1.FeatureGates{ + HTTPEncryption: true, + ServiceMonitorTLSEndpoints: true, + }, + }, + sm: &monitoringv1.ServiceMonitor{ + Spec: monitoringv1.ServiceMonitorSpec{ + Endpoints: []monitoringv1.Endpoint{ + { + TLSConfig: &monitoringv1.TLSConfig{ + CAFile: "/path/to/ca/file", + CertFile: "/path/to/cert/file", + KeyFile: "/path/to/key/file", + }, + BearerTokenSecret: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "abcd-gateway-token", + }, + Key: corev1.ServiceAccountTokenKey, + }, + }, + }, + }, + }, + want: &monitoringv1.ServiceMonitor{ + Spec: monitoringv1.ServiceMonitorSpec{ + Endpoints: []monitoringv1.Endpoint{ + { + TLSConfig: &monitoringv1.TLSConfig{ + CAFile: "/path/to/ca/file", + CertFile: "/path/to/cert/file", + KeyFile: "/path/to/key/file", + }, + BearerTokenSecret: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "abcd-gateway-token", + }, + Key: corev1.ServiceAccountTokenKey, + }, + }, + { + Port: openshift.GatewayOPAInternalPortName, + Path: "/metrics", + Scheme: "https", + BearerTokenSecret: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "abcd-gateway-token", + }, + Key: corev1.ServiceAccountTokenKey, + }, + TLSConfig: &monitoringv1.TLSConfig{ + CAFile: "/path/to/ca/file", + CertFile: "/path/to/cert/file", + KeyFile: "/path/to/key/file", + }, + }, + }, + }, + }, + }, + { + desc: "openshift-network mode with-tls-service-monitor-config", + mode: lokiv1.OpenshiftNetwork, + opts: Options{ + Name: "abcd", + Namespace: "ns", + Stack: lokiv1.LokiStackSpec{ + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.OpenshiftNetwork, + }, + }, + Gates: configv1.FeatureGates{ + HTTPEncryption: true, + ServiceMonitorTLSEndpoints: true, + }, }, sm: &monitoringv1.ServiceMonitor{ Spec: monitoringv1.ServiceMonitorSpec{ @@ -1494,6 +932,12 @@ func TestConfigureServiceMonitorForMode(t *testing.T) { CertFile: "/path/to/cert/file", KeyFile: "/path/to/key/file", }, + BearerTokenSecret: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "abcd-gateway-token", + }, + Key: corev1.ServiceAccountTokenKey, + }, }, }, }, @@ -1507,12 +951,23 @@ func TestConfigureServiceMonitorForMode(t *testing.T) { CertFile: "/path/to/cert/file", KeyFile: "/path/to/key/file", }, + BearerTokenSecret: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "abcd-gateway-token", + }, + Key: corev1.ServiceAccountTokenKey, + }, }, { - Port: openshift.GatewayOPAInternalPortName, - Path: "/metrics", - Scheme: "https", - BearerTokenFile: BearerTokenFile, + Port: openshift.GatewayOPAInternalPortName, + Path: "/metrics", + Scheme: "https", + BearerTokenSecret: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "abcd-gateway-token", + }, + Key: corev1.ServiceAccountTokenKey, + }, TLSConfig: &monitoringv1.TLSConfig{ CAFile: "/path/to/ca/file", CertFile: "/path/to/cert/file", @@ -1528,7 +983,7 @@ func TestConfigureServiceMonitorForMode(t *testing.T) { tc := tc t.Run(tc.desc, func(t *testing.T) { t.Parallel() - err := configureGatewayServiceMonitorForMode(tc.sm, tc.mode, tc.featureGates) + err := configureGatewayServiceMonitorForMode(tc.sm, tc.opts) require.NoError(t, err) require.Equal(t, tc.want, tc.sm) }) diff --git a/operator/internal/manifests/gateway_test.go b/operator/internal/manifests/gateway_test.go index 82cd8003a234..e8df8722fce2 100644 --- a/operator/internal/manifests/gateway_test.go +++ b/operator/internal/manifests/gateway_test.go @@ -2,11 +2,13 @@ package manifests import ( "math/rand" + "path" "reflect" "testing" configv1 "github.com/grafana/loki/operator/apis/config/v1" lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/internal/manifests/internal/gateway" "github.com/grafana/loki/operator/internal/manifests/openshift" "github.com/google/uuid" @@ -49,6 +51,39 @@ func TestNewGatewayDeployment_HasTemplateConfigHashAnnotation(t *testing.T) { require.Equal(t, annotations[expected], sha1C) } +func TestNewGatewayDeployment_HasTemplateCertRotationRequiredAtAnnotation(t *testing.T) { + sha1C := "deadbeef" + ss := NewGatewayDeployment(Options{ + Name: "abcd", + Namespace: "efgh", + CertRotationRequiredAt: "deadbeef", + Stack: lokiv1.LokiStackSpec{ + Template: &lokiv1.LokiTemplateSpec{ + Compactor: &lokiv1.LokiComponentSpec{ + Replicas: rand.Int31(), + }, + Distributor: &lokiv1.LokiComponentSpec{ + Replicas: rand.Int31(), + }, + Ingester: &lokiv1.LokiComponentSpec{ + Replicas: rand.Int31(), + }, + Querier: &lokiv1.LokiComponentSpec{ + Replicas: rand.Int31(), + }, + QueryFrontend: &lokiv1.LokiComponentSpec{ + Replicas: rand.Int31(), + }, + }, + }, + }, sha1C) + + expected := "loki.grafana.com/certRotationRequiredAt" + annotations := ss.Spec.Template.Annotations + require.Contains(t, annotations, expected) + require.Equal(t, annotations[expected], "deadbeef") +} + func TestGatewayConfigMap_ReturnsSHA1OfBinaryContents(t *testing.T) { opts := Options{ Name: uuid.New().String(), @@ -166,7 +201,7 @@ func TestBuildGateway_HasExtraObjectsForTenantMode(t *testing.T) { }) require.NoError(t, err) - require.Len(t, objs, 9) + require.Len(t, objs, 11) } func TestBuildGateway_WithExtraObjectsForTenantMode_RouteSvcMatches(t *testing.T) { @@ -199,8 +234,8 @@ func TestBuildGateway_WithExtraObjectsForTenantMode_RouteSvcMatches(t *testing.T require.NoError(t, err) - svc := objs[2].(*corev1.Service) - rt := objs[3].(*routev1.Route) + svc := objs[4].(*corev1.Service) + rt := objs[5].(*routev1.Route) require.Equal(t, svc.Kind, rt.Spec.To.Kind) require.Equal(t, svc.Name, rt.Spec.To.Name) require.Equal(t, svc.Spec.Ports[0].Name, rt.Spec.Port.TargetPort.StrVal) @@ -237,7 +272,7 @@ func TestBuildGateway_WithExtraObjectsForTenantMode_ServiceAccountNameMatches(t require.NoError(t, err) dpl := objs[1].(*appsv1.Deployment) - sa := objs[4].(*corev1.ServiceAccount) + sa := objs[2].(*corev1.ServiceAccount) require.Equal(t, dpl.Spec.Template.Spec.ServiceAccountName, sa.Name) } @@ -627,3 +662,183 @@ func TestBuildGateway_WithRulesEnabled(t *testing.T) { }) } } + +func TestBuildGateway_WithHTTPEncryption(t *testing.T) { + objs, err := BuildGateway(Options{ + Name: "abcd", + Namespace: "efgh", + Gates: configv1.FeatureGates{ + LokiStackGateway: true, + HTTPEncryption: true, + }, + Stack: lokiv1.LokiStackSpec{ + Template: &lokiv1.LokiTemplateSpec{ + Gateway: &lokiv1.LokiComponentSpec{ + Replicas: rand.Int31(), + }, + Ruler: &lokiv1.LokiComponentSpec{ + Replicas: rand.Int31(), + }, + }, + Rules: &lokiv1.RulesSpec{ + Enabled: true, + }, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.Static, + Authorization: &lokiv1.AuthorizationSpec{}, + Authentication: []lokiv1.AuthenticationSpec{}, + }, + }, + }) + + require.NoError(t, err) + + dpl := objs[1].(*appsv1.Deployment) + require.NotNil(t, dpl) + require.Len(t, dpl.Spec.Template.Spec.Containers, 1) + + c := dpl.Spec.Template.Spec.Containers[0] + + expectedArgs := []string{ + "--debug.name=lokistack-gateway", + "--web.listen=0.0.0.0:8080", + "--web.internal.listen=0.0.0.0:8081", + "--web.healthchecks.url=https://localhost:8080", + "--log.level=warn", + "--logs.read.endpoint=https://abcd-query-frontend-http.efgh.svc.cluster.local:3100", + "--logs.tail.endpoint=https://abcd-query-frontend-http.efgh.svc.cluster.local:3100", + "--logs.write.endpoint=https://abcd-distributor-http.efgh.svc.cluster.local:3100", + "--rbac.config=/etc/lokistack-gateway/rbac.yaml", + "--tenants.config=/etc/lokistack-gateway/tenants.yaml", + "--logs.rules.endpoint=https://abcd-ruler-http.efgh.svc.cluster.local:3100", + "--logs.rules.read-only=true", + "--tls.client-auth-type=NoClientCert", + "--tls.min-version=VersionTLS12", + "--tls.server.cert-file=/var/run/tls/http/server/tls.crt", + "--tls.server.key-file=/var/run/tls/http/server/tls.key", + "--tls.healthchecks.server-ca-file=/var/run/ca/server/service-ca.crt", + "--tls.healthchecks.server-name=abcd-gateway-http.efgh.svc.cluster.local", + "--tls.internal.server.cert-file=/var/run/tls/http/server/tls.crt", + "--tls.internal.server.key-file=/var/run/tls/http/server/tls.key", + "--tls.min-version=", + "--tls.cipher-suites=", + "--logs.tls.ca-file=/var/run/ca/upstream/service-ca.crt", + "--logs.tls.cert-file=/var/run/tls/http/upstream/tls.crt", + "--logs.tls.key-file=/var/run/tls/http/upstream/tls.key", + } + require.Equal(t, expectedArgs, c.Args) + + expectedVolumeMounts := []corev1.VolumeMount{ + { + Name: "rbac", + ReadOnly: true, + MountPath: path.Join(gateway.LokiGatewayMountDir, gateway.LokiGatewayRbacFileName), + SubPath: "rbac.yaml", + }, + { + Name: "tenants", + ReadOnly: true, + MountPath: path.Join(gateway.LokiGatewayMountDir, gateway.LokiGatewayTenantFileName), + SubPath: "tenants.yaml", + }, + { + Name: "lokistack-gateway", + ReadOnly: true, + MountPath: path.Join(gateway.LokiGatewayMountDir, gateway.LokiGatewayRegoFileName), + SubPath: "lokistack-gateway.rego", + }, + { + Name: "tls-secret", + ReadOnly: true, + MountPath: "/var/run/tls/http/server", + }, + { + Name: "abcd-gateway-client-http", + ReadOnly: true, + MountPath: "/var/run/tls/http/upstream", + }, + { + Name: "abcd-ca-bundle", + ReadOnly: true, + MountPath: "/var/run/ca/upstream", + }, + { + Name: "abcd-gateway-ca-bundle", + ReadOnly: true, + MountPath: "/var/run/ca/server", + }, + } + require.Equal(t, expectedVolumeMounts, c.VolumeMounts) + + expectedVolumes := []corev1.Volume{ + { + Name: "rbac", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "abcd-gateway", + }, + }, + }, + }, + { + Name: "tenants", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "abcd-gateway", + }, + }, + }, + }, + { + Name: "lokistack-gateway", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "abcd-gateway", + }, + }, + }, + }, + { + Name: "tls-secret", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "abcd-gateway-http", + }, + }, + }, + { + Name: "abcd-gateway-client-http", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "abcd-gateway-client-http", + }, + }, + }, + { + Name: "abcd-ca-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: &defaultConfigMapMode, + LocalObjectReference: corev1.LocalObjectReference{ + Name: "abcd-ca-bundle", + }, + }, + }, + }, + { + Name: "abcd-gateway-ca-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: &defaultConfigMapMode, + LocalObjectReference: corev1.LocalObjectReference{ + Name: "abcd-gateway-ca-bundle", + }, + }, + }, + }, + } + require.Equal(t, expectedVolumes, dpl.Spec.Template.Spec.Volumes) +} diff --git a/operator/internal/manifests/indexgateway.go b/operator/internal/manifests/indexgateway.go index 5c3a68593052..49ed7afdd7ec 100644 --- a/operator/internal/manifests/indexgateway.go +++ b/operator/internal/manifests/indexgateway.go @@ -36,6 +36,13 @@ func BuildIndexGateway(opts Options) ([]client.Object, error) { } } + if opts.Gates.HTTPEncryption || opts.Gates.GRPCEncryption { + caBundleName := signingCABundleName(opts.Name) + if err := configureServiceCA(&statefulSet.Spec.Template.Spec, caBundleName); err != nil { + return nil, err + } + } + return []client.Object{ statefulSet, NewIndexGatewayGRPCService(opts), @@ -121,7 +128,7 @@ func NewIndexGatewayStatefulSet(opts Options) *appsv1.StatefulSet { } l := ComponentLabels(LabelIndexGatewayComponent, opts.Name) - a := commonAnnotations(opts.ConfigSHA1) + a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) return &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ @@ -183,9 +190,8 @@ func NewIndexGatewayGRPCService(opts Options) *corev1.Service { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: labels, - Annotations: serviceAnnotations(serviceName, opts.Gates.OpenShift.ServingCertsService), + Name: serviceName, + Labels: labels, }, Spec: corev1.ServiceSpec{ ClusterIP: "None", @@ -213,9 +219,8 @@ func NewIndexGatewayHTTPService(opts Options) *corev1.Service { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: labels, - Annotations: serviceAnnotations(serviceName, opts.Gates.OpenShift.ServingCertsService), + Name: serviceName, + Labels: labels, }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ @@ -233,7 +238,7 @@ func NewIndexGatewayHTTPService(opts Options) *corev1.Service { func configureIndexGatewayHTTPServicePKI(statefulSet *appsv1.StatefulSet, opts Options) error { serviceName := serviceNameIndexGatewayHTTP(opts.Name) - return configureHTTPServicePKI(&statefulSet.Spec.Template.Spec, serviceName) + return configureHTTPServicePKI(&statefulSet.Spec.Template.Spec, serviceName, opts.TLSProfile.MinTLSVersion, opts.TLSCipherSuites()) } func configureIndexGatewayGRPCServicePKI(sts *appsv1.StatefulSet, opts Options) error { diff --git a/operator/internal/manifests/indexgateway_test.go b/operator/internal/manifests/indexgateway_test.go index aec2df5cbf98..48499e4b196a 100644 --- a/operator/internal/manifests/indexgateway_test.go +++ b/operator/internal/manifests/indexgateway_test.go @@ -29,6 +29,26 @@ func TestNewIndexGatewayStatefulSet_HasTemplateConfigHashAnnotation(t *testing.T require.Equal(t, annotations[expected], "deadbeef") } +func TestNewIndexGatewayStatefulSet_HasTemplateCertRotationRequiredAtAnnotation(t *testing.T) { + ss := manifests.NewIndexGatewayStatefulSet(manifests.Options{ + Name: "abcd", + Namespace: "efgh", + CertRotationRequiredAt: "deadbeef", + Stack: lokiv1.LokiStackSpec{ + StorageClassName: "standard", + Template: &lokiv1.LokiTemplateSpec{ + IndexGateway: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + }) + expected := "loki.grafana.com/certRotationRequiredAt" + annotations := ss.Spec.Template.Annotations + require.Contains(t, annotations, expected) + require.Equal(t, annotations[expected], "deadbeef") +} + func TestNewIndexGatewayStatefulSet_SelectorMatchesLabels(t *testing.T) { // You must set the .spec.selector field of a StatefulSet to match the labels of // its .spec.template.metadata.labels. Prior to Kubernetes 1.8, the diff --git a/operator/internal/manifests/ingester.go b/operator/internal/manifests/ingester.go index 58ab9c98b853..1eb5e5ec4ef9 100644 --- a/operator/internal/manifests/ingester.go +++ b/operator/internal/manifests/ingester.go @@ -38,6 +38,13 @@ func BuildIngester(opts Options) ([]client.Object, error) { } } + if opts.Gates.HTTPEncryption || opts.Gates.GRPCEncryption { + caBundleName := signingCABundleName(opts.Name) + if err := configureServiceCA(&statefulSet.Spec.Template.Spec, caBundleName); err != nil { + return nil, err + } + } + return []client.Object{ statefulSet, NewIngesterGRPCService(opts), @@ -133,7 +140,7 @@ func NewIngesterStatefulSet(opts Options) *appsv1.StatefulSet { } l := ComponentLabels(LabelIngesterComponent, opts.Name) - a := commonAnnotations(opts.ConfigSHA1) + a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) return &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ Kind: "StatefulSet", @@ -204,7 +211,6 @@ func NewIngesterStatefulSet(opts Options) *appsv1.StatefulSet { // NewIngesterGRPCService creates a k8s service for the ingester GRPC endpoint func NewIngesterGRPCService(opts Options) *corev1.Service { - serviceName := serviceNameIngesterGRPC(opts.Name) labels := ComponentLabels(LabelIngesterComponent, opts.Name) return &corev1.Service{ @@ -213,9 +219,8 @@ func NewIngesterGRPCService(opts Options) *corev1.Service { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceNameIngesterGRPC(opts.Name), - Labels: labels, - Annotations: serviceAnnotations(serviceName, opts.Gates.OpenShift.ServingCertsService), + Name: serviceNameIngesterGRPC(opts.Name), + Labels: labels, }, Spec: corev1.ServiceSpec{ ClusterIP: "None", @@ -243,9 +248,8 @@ func NewIngesterHTTPService(opts Options) *corev1.Service { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: labels, - Annotations: serviceAnnotations(serviceName, opts.Gates.OpenShift.ServingCertsService), + Name: serviceName, + Labels: labels, }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ @@ -263,54 +267,31 @@ func NewIngesterHTTPService(opts Options) *corev1.Service { func configureIngesterHTTPServicePKI(statefulSet *appsv1.StatefulSet, opts Options) error { serviceName := serviceNameIngesterHTTP(opts.Name) - return configureHTTPServicePKI(&statefulSet.Spec.Template.Spec, serviceName) + return configureHTTPServicePKI(&statefulSet.Spec.Template.Spec, serviceName, opts.TLSProfile.MinTLSVersion, opts.TLSCipherSuites()) } func configureIngesterGRPCServicePKI(sts *appsv1.StatefulSet, opts Options) error { - caBundleName := signingCABundleName(opts.Name) - secretVolumeSpec := corev1.PodSpec{ - Volumes: []corev1.Volume{ - { - Name: caBundleName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: caBundleName, - }, - }, - }, - }, - }, - } - secretContainerSpec := corev1.Container{ - VolumeMounts: []corev1.VolumeMount{ - { - Name: caBundleName, - ReadOnly: false, - MountPath: caBundleDir, - }, - }, Args: []string{ // Enable GRPC over TLS for ingester client "-ingester.client.tls-enabled=true", fmt.Sprintf("-ingester.client.tls-cipher-suites=%s", opts.TLSCipherSuites()), fmt.Sprintf("-ingester.client.tls-min-version=%s", opts.TLSProfile.MinTLSVersion), fmt.Sprintf("-ingester.client.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-ingester.client.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-ingester.client.tls-key-path=%s", lokiServerGRPCTLSKey()), fmt.Sprintf("-ingester.client.tls-server-name=%s", fqdn(serviceNameIngesterGRPC(opts.Name), opts.Namespace)), // Enable GRPC over TLS for boltb-shipper index-gateway client "-boltdb.shipper.index-gateway-client.grpc.tls-enabled=true", fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-cipher-suites=%s", opts.TLSCipherSuites()), fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-min-version=%s", opts.TLSProfile.MinTLSVersion), fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-key-path=%s", lokiServerGRPCTLSKey()), fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-server-name=%s", fqdn(serviceNameIndexGatewayGRPC(opts.Name), opts.Namespace)), }, } - if err := mergo.Merge(&sts.Spec.Template.Spec, secretVolumeSpec, mergo.WithAppendSlice); err != nil { - return kverrors.Wrap(err, "failed to merge volumes") - } - if err := mergo.Merge(&sts.Spec.Template.Spec.Containers[0], secretContainerSpec, mergo.WithAppendSlice); err != nil { return kverrors.Wrap(err, "failed to merge container") } diff --git a/operator/internal/manifests/ingester_test.go b/operator/internal/manifests/ingester_test.go index 84314226b59b..657c7aadc298 100644 --- a/operator/internal/manifests/ingester_test.go +++ b/operator/internal/manifests/ingester_test.go @@ -29,6 +29,26 @@ func TestNewIngesterStatefulSet_HasTemplateConfigHashAnnotation(t *testing.T) { require.Equal(t, annotations[expected], "deadbeef") } +func TestNewIngesterStatefulSet_HasTemplateCertRotationRequiredAtAnnotation(t *testing.T) { + ss := manifests.NewIngesterStatefulSet(manifests.Options{ + Name: "abcd", + Namespace: "efgh", + CertRotationRequiredAt: "deadbeef", + Stack: lokiv1.LokiStackSpec{ + StorageClassName: "standard", + Template: &lokiv1.LokiTemplateSpec{ + Ingester: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + }) + expected := "loki.grafana.com/certRotationRequiredAt" + annotations := ss.Spec.Template.Annotations + require.Contains(t, annotations, expected) + require.Equal(t, annotations[expected], "deadbeef") +} + func TestNewIngesterStatefulSet_SelectorMatchesLabels(t *testing.T) { // You must set the .spec.selector field of a StatefulSet to match the labels of // its .spec.template.metadata.labels. Prior to Kubernetes 1.8, the diff --git a/operator/internal/manifests/mutate.go b/operator/internal/manifests/mutate.go index f9bcde48d729..6ba0c77c891c 100644 --- a/operator/internal/manifests/mutate.go +++ b/operator/internal/manifests/mutate.go @@ -23,7 +23,7 @@ import ( // - Deployment // - StatefulSet // - ServiceMonitor -func MutateFuncFor(existing, desired client.Object) controllerutil.MutateFn { +func MutateFuncFor(existing, desired client.Object, depAnnotations map[string]string) controllerutil.MutateFn { return func() error { existingAnnotations := existing.GetAnnotations() if err := mergeWithOverride(&existingAnnotations, desired.GetAnnotations()); err != nil { @@ -47,6 +47,16 @@ func MutateFuncFor(existing, desired client.Object) controllerutil.MutateFn { wantCm := desired.(*corev1.ConfigMap) mutateConfigMap(cm, wantCm) + case *corev1.Secret: + s := existing.(*corev1.Secret) + wantS := desired.(*corev1.Secret) + mutateSecret(s, wantS) + existingAnnotations := s.GetAnnotations() + if err := mergeWithOverride(&existingAnnotations, depAnnotations); err != nil { + return err + } + s.SetAnnotations(existingAnnotations) + case *corev1.Service: svc := existing.(*corev1.Service) wantSvc := desired.(*corev1.Service) @@ -124,10 +134,18 @@ func mergeWithOverride(dst, src interface{}) error { } func mutateConfigMap(existing, desired *corev1.ConfigMap) { + existing.Annotations = desired.Annotations + existing.Labels = desired.Labels existing.BinaryData = desired.BinaryData existing.Data = desired.Data } +func mutateSecret(existing, desired *corev1.Secret) { + existing.Annotations = desired.Annotations + existing.Labels = desired.Labels + existing.Data = desired.Data +} + func mutateServiceAccount(existing, desired *corev1.ServiceAccount) { existing.Annotations = desired.Annotations existing.Labels = desired.Labels @@ -160,6 +178,10 @@ func mutateRoleBinding(existing, desired *rbacv1.RoleBinding) { func mutateServiceMonitor(existing, desired *monitoringv1.ServiceMonitor) { // ServiceMonitor selector is immutable so we set this value only if // a new object is going to be created + existing.Annotations = desired.Annotations + existing.Labels = desired.Labels + existing.Spec.Endpoints = desired.Spec.Endpoints + existing.Spec.JobLabel = desired.Spec.JobLabel } func mutateIngress(existing, desired *networkingv1.Ingress) { diff --git a/operator/internal/manifests/mutate_test.go b/operator/internal/manifests/mutate_test.go index 419a01a86e49..08c8583dc7ce 100644 --- a/operator/internal/manifests/mutate_test.go +++ b/operator/internal/manifests/mutate_test.go @@ -40,7 +40,7 @@ func TestGetMutateFunc_MutateObjectMeta(t *testing.T) { } got := &corev1.ConfigMap{} - f := manifests.MutateFuncFor(got, want) + f := manifests.MutateFuncFor(got, want, nil) err := f() require.NoError(t, err) @@ -53,7 +53,7 @@ func TestGetMutateFunc_MutateObjectMeta(t *testing.T) { func TestGetMutateFunc_ReturnErrOnNotSupportedType(t *testing.T) { got := &corev1.Endpoints{} want := &corev1.Endpoints{} - f := manifests.MutateFuncFor(got, want) + f := manifests.MutateFuncFor(got, want, nil) require.Error(t, f()) } @@ -69,7 +69,7 @@ func TestGetMutateFunc_MutateConfigMap(t *testing.T) { BinaryData: map[string][]byte{"btest": []byte("btestss")}, } - f := manifests.MutateFuncFor(got, want) + f := manifests.MutateFuncFor(got, want, nil) err := f() require.NoError(t, err) @@ -116,7 +116,7 @@ func TestGetMutateFunc_MutateServiceSpec(t *testing.T) { }, } - f := manifests.MutateFuncFor(got, want) + f := manifests.MutateFuncFor(got, want, nil) err := f() require.NoError(t, err) @@ -231,7 +231,7 @@ func TestGetMutateFunc_MutateServiceAccountObjectMeta(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() - f := manifests.MutateFuncFor(tt.got, tt.want) + f := manifests.MutateFuncFor(tt.got, tt.want, nil) err := f() require.NoError(t, err) @@ -293,7 +293,7 @@ func TestGetMutateFunc_MutateClusterRole(t *testing.T) { }, } - f := manifests.MutateFuncFor(got, want) + f := manifests.MutateFuncFor(got, want, nil) err := f() require.NoError(t, err) @@ -358,7 +358,7 @@ func TestGetMutateFunc_MutateClusterRoleBinding(t *testing.T) { }, } - f := manifests.MutateFuncFor(got, want) + f := manifests.MutateFuncFor(got, want, nil) err := f() require.NoError(t, err) @@ -413,7 +413,7 @@ func TestGetMutateFunc_MutateRole(t *testing.T) { }, } - f := manifests.MutateFuncFor(got, want) + f := manifests.MutateFuncFor(got, want, nil) err := f() require.NoError(t, err) @@ -478,7 +478,7 @@ func TestGetMutateFunc_MutateRoleBinding(t *testing.T) { }, } - f := manifests.MutateFuncFor(got, want) + f := manifests.MutateFuncFor(got, want, nil) err := f() require.NoError(t, err) @@ -597,7 +597,7 @@ func TestGeMutateFunc_MutateDeploymentSpec(t *testing.T) { tst := tst t.Run(tst.name, func(t *testing.T) { t.Parallel() - f := manifests.MutateFuncFor(tst.got, tst.want) + f := manifests.MutateFuncFor(tst.got, tst.want, nil) err := f() require.NoError(t, err) @@ -754,7 +754,7 @@ func TestGeMutateFunc_MutateStatefulSetSpec(t *testing.T) { tst := tst t.Run(tst.name, func(t *testing.T) { t.Parallel() - f := manifests.MutateFuncFor(tst.got, tst.want) + f := manifests.MutateFuncFor(tst.got, tst.want, nil) err := f() require.NoError(t, err) @@ -881,7 +881,11 @@ func TestGetMutateFunc_MutateServiceMonitorSpec(t *testing.T) { }, }, want: &monitoringv1.ServiceMonitor{ - ObjectMeta: metav1.ObjectMeta{CreationTimestamp: metav1.Now()}, + ObjectMeta: metav1.ObjectMeta{ + CreationTimestamp: metav1.Now(), + Labels: map[string]string{"test": "label"}, + Annotations: map[string]string{"test": "annotations"}, + }, Spec: monitoringv1.ServiceMonitorSpec{ JobLabel: "some-job-new", Endpoints: []monitoringv1.Endpoint{ @@ -927,14 +931,18 @@ func TestGetMutateFunc_MutateServiceMonitorSpec(t *testing.T) { tst := tst t.Run(tst.name, func(t *testing.T) { t.Parallel() - f := manifests.MutateFuncFor(tst.got, tst.want) + f := manifests.MutateFuncFor(tst.got, tst.want, nil) err := f() require.NoError(t, err) // Ensure not mutated - require.NotEqual(t, tst.got.Spec.JobLabel, tst.want.Spec.JobLabel) - require.NotEqual(t, tst.got.Spec.Endpoints, tst.want.Spec.Endpoints) + require.Equal(t, tst.got.Annotations, tst.want.Annotations) + require.Equal(t, tst.got.Labels, tst.want.Labels) + require.Equal(t, tst.got.Spec.Endpoints, tst.want.Spec.Endpoints) + require.Equal(t, tst.got.Spec.JobLabel, tst.want.Spec.JobLabel) + require.Equal(t, tst.got.Spec.Endpoints, tst.want.Spec.Endpoints) require.NotEqual(t, tst.got.Spec.NamespaceSelector, tst.want.Spec.NamespaceSelector) + require.NotEqual(t, tst.got.Spec.Selector, tst.want.Spec.Selector) }) } } @@ -995,7 +1003,7 @@ func TestGetMutateFunc_MutateIngress(t *testing.T) { }, } - f := manifests.MutateFuncFor(got, want) + f := manifests.MutateFuncFor(got, want, nil) err := f() require.NoError(t, err) @@ -1047,8 +1055,8 @@ func TestGetMutateFunc_MutateRoute(t *testing.T) { }, }, } + f := manifests.MutateFuncFor(got, want, nil) - f := manifests.MutateFuncFor(got, want) err := f() require.NoError(t, err) diff --git a/operator/internal/manifests/openshift/build.go b/operator/internal/manifests/openshift/build.go index f203c621a160..4d9234891009 100644 --- a/operator/internal/manifests/openshift/build.go +++ b/operator/internal/manifests/openshift/build.go @@ -9,7 +9,7 @@ import ( func BuildGatewayObjects(opts Options) []client.Object { return []client.Object{ BuildRoute(opts), - BuildGatewayServiceAccount(opts), + BuildGatewayCAConfigMap(opts), BuildGatewayClusterRole(opts), BuildGatewayClusterRoleBinding(opts), BuildMonitoringRole(opts), @@ -17,18 +17,11 @@ func BuildGatewayObjects(opts Options) []client.Object { } } -// BuildLokiStackObjects returns a list of auxiliary openshift/k8s objects -// for lokistack deployments on OpenShift. -func BuildLokiStackObjects(opts Options) []client.Object { - return []client.Object{ - BuildServiceCAConfigMap(opts), - } -} - // BuildRulerObjects returns a list of auxiliary openshift/k8s objects // for lokistack ruler deployments on OpenShift. func BuildRulerObjects(opts Options) []client.Object { return []client.Object{ + BuildAlertManagerCAConfigMap(opts), BuildRulerServiceAccount(opts), BuildRulerClusterRole(opts), BuildRulerClusterRoleBinding(opts), diff --git a/operator/internal/manifests/openshift/build_test.go b/operator/internal/manifests/openshift/build_test.go index 1180383d9107..16170d28b905 100644 --- a/operator/internal/manifests/openshift/build_test.go +++ b/operator/internal/manifests/openshift/build_test.go @@ -1,29 +1,15 @@ package openshift import ( - "encoding/json" "testing" "github.com/stretchr/testify/require" lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" - routev1 "github.com/openshift/api/route/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" ) -func TestBuildGatewayObjects_ServiceAccountRefMatches(t *testing.T) { - opts := NewOptions(lokiv1.OpenshiftLogging, "abc", "ns", "abc", "example.com", "abc", "abc", map[string]string{}, map[string]TenantData{}, "abc") - - objs := BuildGatewayObjects(opts) - sa := objs[1].(*corev1.ServiceAccount) - rb := objs[3].(*rbacv1.ClusterRoleBinding) - - require.Equal(t, sa.Kind, rb.Subjects[0].Kind) - require.Equal(t, sa.Name, rb.Subjects[0].Name) - require.Equal(t, sa.Namespace, rb.Subjects[0].Namespace) -} - func TestBuildGatewayObjects_ClusterRoleRefMatches(t *testing.T) { opts := NewOptions(lokiv1.OpenshiftLogging, "abc", "ns", "abc", "example.com", "abc", "abc", map[string]string{}, map[string]TenantData{}, "abc") @@ -46,39 +32,13 @@ func TestBuildGatewayObjects_MonitoringClusterRoleRefMatches(t *testing.T) { require.Equal(t, cr.Name, rb.RoleRef.Name) } -func TestBuildGatewayObjects_ServiceAccountAnnotationsRouteRefMatches(t *testing.T) { - opts := NewOptions(lokiv1.OpenshiftLogging, "abc", "ns", "abc", "example.com", "abc", "abc", map[string]string{}, map[string]TenantData{}, "abc") - - objs := BuildGatewayObjects(opts) - rt := objs[0].(*routev1.Route) - sa := objs[1].(*corev1.ServiceAccount) - - type oauthRedirectReference struct { - Kind string `json:"kind"` - APIVersion string `json:"apiVersion"` - Ref *struct { - Kind string `json:"kind"` - Name string `json:"name"` - } `json:"reference"` - } - - for _, a := range sa.Annotations { - oauthRef := oauthRedirectReference{} - err := json.Unmarshal([]byte(a), &oauthRef) - require.NoError(t, err) - - require.Equal(t, rt.Name, oauthRef.Ref.Name) - require.Equal(t, rt.Kind, oauthRef.Ref.Kind) - } -} - -func TestBuildRulerObjects(t *testing.T) { +func TestBuildRulerObjects_ClusterRoleRefMatches(t *testing.T) { opts := NewOptions(lokiv1.OpenshiftLogging, "abc", "ns", "abc", "example.com", "abc", "abc", map[string]string{}, map[string]TenantData{}, "abc") objs := BuildRulerObjects(opts) - sa := objs[0].(*corev1.ServiceAccount) - cr := objs[1].(*rbacv1.ClusterRole) - rb := objs[2].(*rbacv1.ClusterRoleBinding) + sa := objs[1].(*corev1.ServiceAccount) + cr := objs[2].(*rbacv1.ClusterRole) + rb := objs[3].(*rbacv1.ClusterRoleBinding) require.Equal(t, sa.Kind, rb.Subjects[0].Kind) require.Equal(t, sa.Name, rb.Subjects[0].Name) diff --git a/operator/internal/manifests/openshift/configure.go b/operator/internal/manifests/openshift/configure.go index 0b03674c8eab..ab710a548c4c 100644 --- a/operator/internal/manifests/openshift/configure.go +++ b/operator/internal/manifests/openshift/configure.go @@ -2,9 +2,6 @@ package openshift import ( "fmt" - "path" - "regexp" - "strings" "github.com/ViaQ/logerr/v2/kverrors" "github.com/imdario/mergo" @@ -39,8 +36,6 @@ var ( networkTenants = []string{ tenantNetwork, } - - logsEndpointRe = regexp.MustCompile(`.*logs..*.endpoint.*`) ) // GetTenants return the slice of all supported tenants for a specified mode @@ -62,110 +57,18 @@ func GetTenants(mode lokiv1.ModeType) []string { func ConfigureGatewayDeployment( d *appsv1.Deployment, mode lokiv1.ModeType, - gwContainerName string, - secretVolumeName, tlsDir, certFile, keyFile string, - caBundleVolumeName, caDir, caFile string, - withTLS, withCertSigningService bool, - secretName, serverName string, - gatewayHTTPPort int, - minTLSVersion string, - ciphers string, + secretVolumeName, tlsDir string, + minTLSVersion, ciphers string, + withTLS bool, ) error { - var gwIndex int - for i, c := range d.Spec.Template.Spec.Containers { - if c.Name == gwContainerName { - gwIndex = i - break - } - } - - gwContainer := d.Spec.Template.Spec.Containers[gwIndex].DeepCopy() - gwArgs := gwContainer.Args - gwVolumes := d.Spec.Template.Spec.Volumes - - if withCertSigningService { - for i, a := range gwArgs { - if logsEndpointRe.MatchString(a) { - gwContainer.Args[i] = strings.Replace(a, "http", "https", 1) - } - } - - gwArgs = append(gwArgs, fmt.Sprintf("--logs.tls.ca-file=%s/%s", caDir, caFile)) - - gwContainer.VolumeMounts = append(gwContainer.VolumeMounts, corev1.VolumeMount{ - Name: caBundleVolumeName, - ReadOnly: true, - MountPath: caDir, - }) - - gwVolumes = append(gwVolumes, corev1.Volume{ - Name: caBundleVolumeName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - DefaultMode: &defaultConfigMapMode, - LocalObjectReference: corev1.LocalObjectReference{ - Name: caBundleVolumeName, - }, - }, - }, - }) - } - - for i, a := range gwArgs { - if strings.HasPrefix(a, "--web.healthchecks.url=") { - gwArgs[i] = fmt.Sprintf("--web.healthchecks.url=https://localhost:%d", gatewayHTTPPort) - break - } - } - - certFilePath := path.Join(tlsDir, certFile) - keyFilePath := path.Join(tlsDir, keyFile) - caFilePath := path.Join(caDir, caFile) - gwArgs = append(gwArgs, - "--tls.client-auth-type=NoClientCert", - fmt.Sprintf("--tls.server.cert-file=%s", certFilePath), - fmt.Sprintf("--tls.server.key-file=%s", keyFilePath), - fmt.Sprintf("--tls.healthchecks.server-ca-file=%s", caFilePath), - fmt.Sprintf("--tls.healthchecks.server-name=%s", serverName)) - - gwContainer.ReadinessProbe.ProbeHandler.HTTPGet.Scheme = corev1.URISchemeHTTPS - gwContainer.LivenessProbe.ProbeHandler.HTTPGet.Scheme = corev1.URISchemeHTTPS - - // Create and mount TLS secrets volumes if not already created. - if !withTLS { - gwVolumes = append(gwVolumes, corev1.Volume{ - Name: secretVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: secretName, - }, - }, - }) - - gwContainer.VolumeMounts = append(gwContainer.VolumeMounts, corev1.VolumeMount{ - Name: secretVolumeName, - ReadOnly: true, - MountPath: tlsDir, - }) - - // Add TLS profile info args since openshift gateway always uses TLS. - gwArgs = append(gwArgs, - fmt.Sprintf("--tls.min-version=%s", minTLSVersion), - fmt.Sprintf("--tls.cipher-suites=%s", ciphers)) - } - - gwContainer.Args = gwArgs - p := corev1.PodSpec{ ServiceAccountName: d.GetName(), Containers: []corev1.Container{ - *gwContainer, - newOPAOpenShiftContainer(mode, secretVolumeName, tlsDir, certFile, keyFile, minTLSVersion, ciphers, withTLS), + newOPAOpenShiftContainer(mode, secretVolumeName, tlsDir, minTLSVersion, ciphers, withTLS), }, - Volumes: gwVolumes, } - if err := mergo.Merge(&d.Spec.Template.Spec, p, mergo.WithOverride); err != nil { + if err := mergo.Merge(&d.Spec.Template.Spec, p, mergo.WithAppendSlice); err != nil { return kverrors.Wrap(err, "failed to merge sidecar container spec ") } @@ -198,13 +101,15 @@ func ConfigureGatewayServiceMonitor(sm *monitoringv1.ServiceMonitor, withTLS boo var opaEndpoint monitoringv1.Endpoint if withTLS { + bearerTokenSecret := sm.Spec.Endpoints[0].BearerTokenSecret tlsConfig := sm.Spec.Endpoints[0].TLSConfig + opaEndpoint = monitoringv1.Endpoint{ - Port: opaMetricsPortName, - Path: "/metrics", - Scheme: "https", - BearerTokenFile: bearerTokenFile, - TLSConfig: tlsConfig, + Port: opaMetricsPortName, + Path: "/metrics", + Scheme: "https", + BearerTokenSecret: bearerTokenSecret, + TLSConfig: tlsConfig, } } else { opaEndpoint = monitoringv1.Endpoint{ @@ -225,44 +130,30 @@ func ConfigureGatewayServiceMonitor(sm *monitoringv1.ServiceMonitor, withTLS boo return nil } -// ConfigureQueryFrontendDeployment configures use of TLS when enabled. -func ConfigureQueryFrontendDeployment( - d *appsv1.Deployment, - proxyURL string, - qfContainerName string, - caBundleVolumeName, caDir, caFile string, +// ConfigureRulerStatefulSet configures the ruler to use the cluster monitoring alertmanager. +func ConfigureRulerStatefulSet( + ss *appsv1.StatefulSet, + alertmanagerCABundleName string, + token, caDir, caPath string, + monitorServerName, rulerContainerName string, ) error { - var qfIdx int - for i, c := range d.Spec.Template.Spec.Containers { - if c.Name == qfContainerName { - qfIdx = i + var rulerIndex int + for i, c := range ss.Spec.Template.Spec.Containers { + if c.Name == rulerContainerName { + rulerIndex = i break } } - containerSpec := corev1.Container{ - Args: []string{ - fmt.Sprintf("-frontend.tail-proxy-url=%s", proxyURL), - fmt.Sprintf("-frontend.tail-tls-config.tls-ca-path=%s/%s", caDir, caFile), - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: caBundleVolumeName, - ReadOnly: true, - MountPath: caDir, - }, - }, - } - - p := corev1.PodSpec{ + secretVolumeSpec := corev1.PodSpec{ Volumes: []corev1.Volume{ { - Name: caBundleVolumeName, + Name: alertmanagerCABundleName, VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ DefaultMode: &defaultConfigMapMode, LocalObjectReference: corev1.LocalObjectReference{ - Name: caBundleVolumeName, + Name: alertmanagerCABundleName, }, }, }, @@ -270,39 +161,20 @@ func ConfigureQueryFrontendDeployment( }, } - if err := mergo.Merge(&d.Spec.Template.Spec.Containers[qfIdx], containerSpec, mergo.WithAppendSlice); err != nil { - return kverrors.Wrap(err, "failed to add tls config args") - } - - if err := mergo.Merge(&d.Spec.Template.Spec, p, mergo.WithAppendSlice); err != nil { - return kverrors.Wrap(err, "failed to add tls volumes") - } - - return nil -} - -// ConfigureRulerStatefulSet configures the ruler to use the cluster monitoring alertmanager. -func ConfigureRulerStatefulSet( - ss *appsv1.StatefulSet, - token, caBundleVolumeName, caDir, caFile string, - monitorServerName, rulerContainerName string, -) error { - var rulerIndex int - for i, c := range ss.Spec.Template.Spec.Containers { - if c.Name == rulerContainerName { - rulerIndex = i - break - } - } - rulerContainer := ss.Spec.Template.Spec.Containers[rulerIndex].DeepCopy() rulerContainer.Args = append(rulerContainer.Args, - fmt.Sprintf("-ruler.alertmanager-client.tls-ca-path=%s/%s", caDir, caFile), + fmt.Sprintf("-ruler.alertmanager-client.tls-ca-path=%s", caPath), fmt.Sprintf("-ruler.alertmanager-client.tls-server-name=%s", monitorServerName), fmt.Sprintf("-ruler.alertmanager-client.credentials-file=%s", token), ) + rulerContainer.VolumeMounts = append(rulerContainer.VolumeMounts, corev1.VolumeMount{ + Name: alertmanagerCABundleName, + ReadOnly: true, + MountPath: caDir, + }) + p := corev1.PodSpec{ ServiceAccountName: ss.GetName(), Containers: []corev1.Container{ @@ -310,6 +182,10 @@ func ConfigureRulerStatefulSet( }, } + if err := mergo.Merge(&ss.Spec.Template.Spec, secretVolumeSpec, mergo.WithAppendSlice); err != nil { + return kverrors.Wrap(err, "failed to merge volumes") + } + if err := mergo.Merge(&ss.Spec.Template.Spec, p, mergo.WithOverride); err != nil { return kverrors.Wrap(err, "failed to merge ruler container spec ") } diff --git a/operator/internal/manifests/openshift/opa_openshift.go b/operator/internal/manifests/openshift/opa_openshift.go index 40faf3978818..3104a17d2282 100644 --- a/operator/internal/manifests/openshift/opa_openshift.go +++ b/operator/internal/manifests/openshift/opa_openshift.go @@ -21,7 +21,7 @@ const ( opaDefaultLabelMatcher = "kubernetes_namespace_name" ) -func newOPAOpenShiftContainer(mode lokiv1.ModeType, secretVolumeName, tlsDir, certFile, keyFile, minTLSVersion, ciphers string, withTLS bool) corev1.Container { +func newOPAOpenShiftContainer(mode lokiv1.ModeType, secretVolumeName, tlsDir, minTLSVersion, ciphers string, withTLS bool) corev1.Container { var ( image string args []string @@ -52,8 +52,8 @@ func newOPAOpenShiftContainer(mode lokiv1.ModeType, secretVolumeName, tlsDir, ce } if withTLS { - certFilePath := path.Join(tlsDir, certFile) - keyFilePath := path.Join(tlsDir, keyFile) + certFilePath := path.Join(tlsDir, corev1.TLSCertKey) + keyFilePath := path.Join(tlsDir, corev1.TLSPrivateKeyKey) args = append(args, []string{ fmt.Sprintf("--tls.internal.server.cert-file=%s", certFilePath), diff --git a/operator/internal/manifests/openshift/service_ca.go b/operator/internal/manifests/openshift/service_ca.go index bb201d2b761f..247c1e856c0d 100644 --- a/operator/internal/manifests/openshift/service_ca.go +++ b/operator/internal/manifests/openshift/service_ca.go @@ -5,10 +5,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// BuildServiceCAConfigMap returns a k8s configmap for the LokiStack +// BuildGatewayCAConfigMap returns a k8s configmap for the LokiStack // serviceCA configmap. This configmap is used to configure // the gateway and components to verify TLS certificates. -func BuildServiceCAConfigMap(opts Options) *corev1.ConfigMap { +func BuildGatewayCAConfigMap(opts Options) *corev1.ConfigMap { return &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ Kind: "ConfigMap", @@ -24,3 +24,23 @@ func BuildServiceCAConfigMap(opts Options) *corev1.ConfigMap { }, } } + +// BuildAlertManagerCAConfigMap returns a k8s configmap for the LokiStack +// alertmanager serviceCA configmap. This configmap is used to configure +// the ruler to verify AlertManager TLS certificates. +func BuildAlertManagerCAConfigMap(opts Options) *corev1.ConfigMap { + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + InjectCABundleKey: "true", + }, + Labels: opts.BuildOpts.Labels, + Name: alertmanagerCABundleName(opts), + Namespace: opts.BuildOpts.LokiStackNamespace, + }, + } +} diff --git a/operator/internal/manifests/openshift/serviceaccount.go b/operator/internal/manifests/openshift/serviceaccount.go index 82b3805a0ab2..216331cf13a2 100644 --- a/operator/internal/manifests/openshift/serviceaccount.go +++ b/operator/internal/manifests/openshift/serviceaccount.go @@ -7,25 +7,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// BuildGatewayServiceAccount returns a k8s object for the LokiStack Gateway -// serviceaccount. This ServiceAccount is used in parallel as an -// OpenShift OAuth Client. -func BuildGatewayServiceAccount(opts Options) client.Object { - return &corev1.ServiceAccount{ - TypeMeta: metav1.TypeMeta{ - Kind: "ServiceAccount", - APIVersion: corev1.SchemeGroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Annotations: serviceAccountAnnotations(opts), - Labels: opts.BuildOpts.Labels, - Name: gatewayServiceAccountName(opts), - Namespace: opts.BuildOpts.LokiStackNamespace, - }, - AutomountServiceAccountToken: pointer.Bool(true), - } -} - // BuildRulerServiceAccount returns a k8s object for the LokiStack Ruler // serviceaccount. // This ServiceAccount is used to autheticate and access the alertmanager host. @@ -36,10 +17,9 @@ func BuildRulerServiceAccount(opts Options) client.Object { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Annotations: serviceAccountAnnotations(opts), - Labels: opts.BuildOpts.Labels, - Name: rulerServiceAccountName(opts), - Namespace: opts.BuildOpts.LokiStackNamespace, + Labels: opts.BuildOpts.Labels, + Name: rulerServiceAccountName(opts), + Namespace: opts.BuildOpts.LokiStackNamespace, }, AutomountServiceAccountToken: pointer.Bool(true), } diff --git a/operator/internal/manifests/openshift/serviceaccount_test.go b/operator/internal/manifests/openshift/serviceaccount_test.go deleted file mode 100644 index bb857b4b7798..000000000000 --- a/operator/internal/manifests/openshift/serviceaccount_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package openshift - -import ( - "fmt" - "testing" - - lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" - - "github.com/stretchr/testify/require" -) - -func TestBuildServiceAccount_AnnotationsMatchLoggingTenants(t *testing.T) { - opts := NewOptions(lokiv1.OpenshiftLogging, "abc", "ns", "abc", "example.com", "abc", "abc", map[string]string{}, map[string]TenantData{}, "abc") - - sa := BuildGatewayServiceAccount(opts) - require.Len(t, sa.GetAnnotations(), len(loggingTenants)) - - var keys []string - for key := range sa.GetAnnotations() { - keys = append(keys, key) - } - - for _, name := range loggingTenants { - v := fmt.Sprintf("serviceaccounts.openshift.io/oauth-redirectreference.%s", name) - require.Contains(t, keys, v) - } -} - -func TestBuildServiceAccount_AnnotationsMatchNetworkTenants(t *testing.T) { - opts := NewOptions(lokiv1.OpenshiftNetwork, "def", "ns2", "def", "example2.com", "def", "def", map[string]string{}, map[string]TenantData{}, "abc") - - sa := BuildGatewayServiceAccount(opts) - require.Len(t, sa.GetAnnotations(), len(networkTenants)) - - var keys []string - for key := range sa.GetAnnotations() { - keys = append(keys, key) - } - - for _, name := range networkTenants { - v := fmt.Sprintf("serviceaccounts.openshift.io/oauth-redirectreference.%s", name) - require.Contains(t, keys, v) - } -} diff --git a/operator/internal/manifests/openshift/var.go b/operator/internal/manifests/openshift/var.go index 7bac13007e1b..e0711f45d7ab 100644 --- a/operator/internal/manifests/openshift/var.go +++ b/operator/internal/manifests/openshift/var.go @@ -15,8 +15,6 @@ var ( // GatewayOPAInternalPortName is the HTTP container metrics port name of the OpenPolicyAgent sidecar. GatewayOPAInternalPortName = "opa-metrics" - bearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" - cookieSecretLength = 32 allowedRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") @@ -65,10 +63,17 @@ func rulerServiceAccountName(opts Options) string { } func serviceCABundleName(opts Options) string { - return fmt.Sprintf("%s-ca-bundle", opts.BuildOpts.LokiStackName) + return fmt.Sprintf("%s-ca-bundle", opts.BuildOpts.GatewayName) +} + +func alertmanagerCABundleName(opts Options) string { + return fmt.Sprintf("%s-ca-bundle", opts.BuildOpts.RulerName) } -func serviceAccountAnnotations(opts Options) map[string]string { +// ServiceAccountAnnotations returns a map of OpenShift specific routes for ServiceAccounts. +// Specifically the serviceacount will be annotated for each tenant with the OAuthRedirectReference +// to make the serviceaccount a valid oauth-client. +func ServiceAccountAnnotations(opts Options) map[string]string { a := make(map[string]string, len(opts.Authentication)) for _, auth := range opts.Authentication { key := fmt.Sprintf("serviceaccounts.openshift.io/oauth-redirectreference.%s", auth.TenantName) diff --git a/operator/internal/manifests/options.go b/operator/internal/manifests/options.go index ea7c48638d0c..b1857d343103 100644 --- a/operator/internal/manifests/options.go +++ b/operator/internal/manifests/options.go @@ -14,12 +14,13 @@ import ( // Options is a set of configuration values to use when building manifests such as resource sizes, etc. // Most of this should be provided - either directly or indirectly - by the user. type Options struct { - Name string - Namespace string - Image string - GatewayImage string - GatewayBaseDomain string - ConfigSHA1 string + Name string + Namespace string + Image string + GatewayImage string + GatewayBaseDomain string + ConfigSHA1 string + CertRotationRequiredAt string Gates configv1.FeatureGates Stack lokiv1.LokiStackSpec diff --git a/operator/internal/manifests/querier.go b/operator/internal/manifests/querier.go index a0a3cc98c772..be11bf7d2ce6 100644 --- a/operator/internal/manifests/querier.go +++ b/operator/internal/manifests/querier.go @@ -37,6 +37,13 @@ func BuildQuerier(opts Options) ([]client.Object, error) { } } + if opts.Gates.HTTPEncryption || opts.Gates.GRPCEncryption { + caBundleName := signingCABundleName(opts.Name) + if err := configureServiceCA(&deployment.Spec.Template.Spec, caBundleName); err != nil { + return nil, err + } + } + return []client.Object{ deployment, NewQuerierGRPCService(opts), @@ -122,7 +129,7 @@ func NewQuerierDeployment(opts Options) *appsv1.Deployment { } l := ComponentLabels(LabelQuerierComponent, opts.Name) - a := commonAnnotations(opts.ConfigSHA1) + a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ @@ -164,9 +171,8 @@ func NewQuerierGRPCService(opts Options) *corev1.Service { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: labels, - Annotations: serviceAnnotations(serviceName, opts.Gates.OpenShift.ServingCertsService), + Name: serviceName, + Labels: labels, }, Spec: corev1.ServiceSpec{ ClusterIP: "None", @@ -194,9 +200,8 @@ func NewQuerierHTTPService(opts Options) *corev1.Service { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: labels, - Annotations: serviceAnnotations(serviceName, opts.Gates.OpenShift.ServingCertsService), + Name: serviceName, + Labels: labels, }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ @@ -214,60 +219,39 @@ func NewQuerierHTTPService(opts Options) *corev1.Service { func configureQuerierHTTPServicePKI(deployment *appsv1.Deployment, opts Options) error { serviceName := serviceNameQuerierHTTP(opts.Name) - return configureHTTPServicePKI(&deployment.Spec.Template.Spec, serviceName) + return configureHTTPServicePKI(&deployment.Spec.Template.Spec, serviceName, opts.TLSProfile.MinTLSVersion, opts.TLSCipherSuites()) } func configureQuerierGRPCServicePKI(deployment *appsv1.Deployment, opts Options) error { - caBundleName := signingCABundleName(opts.Name) - secretVolumeSpec := corev1.PodSpec{ - Volumes: []corev1.Volume{ - { - Name: caBundleName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: caBundleName, - }, - }, - }, - }, - }, - } - secretContainerSpec := corev1.Container{ - VolumeMounts: []corev1.VolumeMount{ - { - Name: caBundleName, - ReadOnly: false, - MountPath: caBundleDir, - }, - }, Args: []string{ // Enable GRPC over TLS for ingester client "-ingester.client.tls-enabled=true", fmt.Sprintf("-ingester.client.tls-cipher-suites=%s", opts.TLSCipherSuites()), fmt.Sprintf("-ingester.client.tls-min-version=%s", opts.TLSProfile.MinTLSVersion), fmt.Sprintf("-ingester.client.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-ingester.client.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-ingester.client.tls-key-path=%s", lokiServerGRPCTLSKey()), fmt.Sprintf("-ingester.client.tls-server-name=%s", fqdn(serviceNameIngesterGRPC(opts.Name), opts.Namespace)), // Enable GRPC over TLS for query frontend client "-querier.frontend-client.tls-enabled=true", fmt.Sprintf("-querier.frontend-client.tls-cipher-suites=%s", opts.TLSCipherSuites()), fmt.Sprintf("-querier.frontend-client.tls-min-version=%s", opts.TLSProfile.MinTLSVersion), fmt.Sprintf("-querier.frontend-client.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-querier.frontend-client.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-querier.frontend-client.tls-key-path=%s", lokiServerGRPCTLSKey()), fmt.Sprintf("-querier.frontend-client.tls-server-name=%s", fqdn(serviceNameQueryFrontendGRPC(opts.Name), opts.Namespace)), // Enable GRPC over TLS for boltb-shipper index-gateway client "-boltdb.shipper.index-gateway-client.grpc.tls-enabled=true", fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-cipher-suites=%s", opts.TLSCipherSuites()), fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-min-version=%s", opts.TLSProfile.MinTLSVersion), fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-key-path=%s", lokiServerGRPCTLSKey()), fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-server-name=%s", fqdn(serviceNameIndexGatewayGRPC(opts.Name), opts.Namespace)), }, } - if err := mergo.Merge(&deployment.Spec.Template.Spec, secretVolumeSpec, mergo.WithAppendSlice); err != nil { - return kverrors.Wrap(err, "failed to merge volumes") - } - if err := mergo.Merge(&deployment.Spec.Template.Spec.Containers[0], secretContainerSpec, mergo.WithAppendSlice); err != nil { return kverrors.Wrap(err, "failed to merge container") } diff --git a/operator/internal/manifests/querier_test.go b/operator/internal/manifests/querier_test.go index a452f0906825..fe81d32a52f7 100644 --- a/operator/internal/manifests/querier_test.go +++ b/operator/internal/manifests/querier_test.go @@ -29,6 +29,26 @@ func TestNewQuerierDeployment_HasTemplateConfigHashAnnotation(t *testing.T) { require.Equal(t, annotations[expected], "deadbeef") } +func TestNewQuerierDeployment_HasTemplateCertRotationRequiredAtAnnotation(t *testing.T) { + ss := manifests.NewQuerierDeployment(manifests.Options{ + Name: "abcd", + Namespace: "efgh", + CertRotationRequiredAt: "deadbeef", + Stack: lokiv1.LokiStackSpec{ + Template: &lokiv1.LokiTemplateSpec{ + Querier: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + }) + + expected := "loki.grafana.com/certRotationRequiredAt" + annotations := ss.Spec.Template.Annotations + require.Contains(t, annotations, expected) + require.Equal(t, annotations[expected], "deadbeef") +} + func TestNewQuerierDeployment_SelectorMatchesLabels(t *testing.T) { // You must set the .spec.selector field of a Deployment to match the labels of // its .spec.template.metadata.labels. Prior to Kubernetes 1.8, the diff --git a/operator/internal/manifests/query-frontend.go b/operator/internal/manifests/query-frontend.go index 4785db7b77f4..935d061cde19 100644 --- a/operator/internal/manifests/query-frontend.go +++ b/operator/internal/manifests/query-frontend.go @@ -32,6 +32,13 @@ func BuildQueryFrontend(opts Options) ([]client.Object, error) { } } + if opts.Gates.HTTPEncryption || opts.Gates.GRPCEncryption { + caBundleName := signingCABundleName(opts.Name) + if err := configureServiceCA(&deployment.Spec.Template.Spec, caBundleName); err != nil { + return nil, err + } + } + return []client.Object{ deployment, NewQueryFrontendGRPCService(opts), @@ -129,7 +136,7 @@ func NewQueryFrontendDeployment(opts Options) *appsv1.Deployment { } l := ComponentLabels(LabelQueryFrontendComponent, opts.Name) - a := commonAnnotations(opts.ConfigSHA1) + a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) return &appsv1.Deployment{ TypeMeta: metav1.TypeMeta{ @@ -171,9 +178,8 @@ func NewQueryFrontendGRPCService(opts Options) *corev1.Service { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: labels, - Annotations: serviceAnnotations(serviceName, opts.Gates.OpenShift.ServingCertsService), + Name: serviceName, + Labels: labels, }, Spec: corev1.ServiceSpec{ ClusterIP: "None", @@ -201,9 +207,8 @@ func NewQueryFrontendHTTPService(opts Options) *corev1.Service { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: labels, - Annotations: serviceAnnotations(serviceName, opts.Gates.OpenShift.ServingCertsService), + Name: serviceName, + Labels: labels, }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ @@ -220,80 +225,36 @@ func NewQueryFrontendHTTPService(opts Options) *corev1.Service { } func configureQueryFrontendHTTPServicePKI(deployment *appsv1.Deployment, opts Options) error { - serviceName := serviceNameQueryFrontendHTTP(opts.Name) - caBundleName := signingCABundleName(opts.Name) - - err := configureTailCA( - deployment, - lokiFrontendContainerName, - caBundleName, - caBundleDir, - caFile, - opts.TLSProfile.MinTLSVersion, - opts.TLSCipherSuites(), - ) - if err != nil { - return err - } - - return configureHTTPServicePKI(&deployment.Spec.Template.Spec, serviceName) -} - -func configureQueryFrontendGRPCServicePKI(deployment *appsv1.Deployment, opts Options) error { - serviceName := serviceNameQueryFrontendGRPC(opts.Name) - return configureGRPCServicePKI(&deployment.Spec.Template.Spec, serviceName) -} - -// ConfigureQueryFrontendDeployment configures CA certificate when TLS is enabled. -func configureTailCA(d *appsv1.Deployment, - qfContainerName, caBundleVolumeName, caDir, caFile, minTLSVersion, cipherSuites string, -) error { var qfIdx int - for i, c := range d.Spec.Template.Spec.Containers { - if c.Name == qfContainerName { + for i, c := range deployment.Spec.Template.Spec.Containers { + if c.Name == lokiFrontendContainerName { qfIdx = i break } } + url := fmt.Sprintf("https://%s:%d", fqdn(serviceNameQuerierHTTP(opts.Name), opts.Namespace), httpPort) + containerSpec := corev1.Container{ Args: []string{ - fmt.Sprintf("-frontend.tail-tls-config.tls-cipher-suites=%s", cipherSuites), - fmt.Sprintf("-frontend.tail-tls-config.tls-min-version=%s", minTLSVersion), - fmt.Sprintf("-frontend.tail-tls-config.tls-ca-path=%s/%s", caDir, caFile), - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: caBundleVolumeName, - ReadOnly: true, - MountPath: caDir, - }, - }, - } - - p := corev1.PodSpec{ - Volumes: []corev1.Volume{ - { - Name: caBundleVolumeName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - DefaultMode: &defaultConfigMapMode, - LocalObjectReference: corev1.LocalObjectReference{ - Name: caBundleVolumeName, - }, - }, - }, - }, + fmt.Sprintf("-frontend.tail-proxy-url=%s", url), + fmt.Sprintf("-frontend.tail-tls-config.tls-min-version=%s", opts.TLSProfile.MinTLSVersion), + fmt.Sprintf("-frontend.tail-tls-config.tls-cipher-suites=%s", opts.TLSCipherSuites()), + fmt.Sprintf("-frontend.tail-tls-config.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-frontend.tail-tls-config.tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-frontend.tail-tls-config.tls-key-path=%s", lokiServerHTTPTLSKey()), }, } - if err := mergo.Merge(&d.Spec.Template.Spec.Containers[qfIdx], containerSpec, mergo.WithAppendSlice); err != nil { + if err := mergo.Merge(&deployment.Spec.Template.Spec.Containers[qfIdx], containerSpec, mergo.WithAppendSlice); err != nil { return kverrors.Wrap(err, "failed to add tls config args") } - if err := mergo.Merge(&d.Spec.Template.Spec, p, mergo.WithAppendSlice); err != nil { - return kverrors.Wrap(err, "failed to add tls volumes") - } + serviceName := serviceNameQueryFrontendHTTP(opts.Name) + return configureHTTPServicePKI(&deployment.Spec.Template.Spec, serviceName, opts.TLSProfile.MinTLSVersion, opts.TLSCipherSuites()) +} - return nil +func configureQueryFrontendGRPCServicePKI(deployment *appsv1.Deployment, opts Options) error { + serviceName := serviceNameQueryFrontendGRPC(opts.Name) + return configureGRPCServicePKI(&deployment.Spec.Template.Spec, serviceName) } diff --git a/operator/internal/manifests/query-frontend_test.go b/operator/internal/manifests/query-frontend_test.go index fdb811e4e5b6..c31bc1004aef 100644 --- a/operator/internal/manifests/query-frontend_test.go +++ b/operator/internal/manifests/query-frontend_test.go @@ -1,17 +1,10 @@ package manifests import ( - "fmt" - "path" "testing" lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" - "github.com/grafana/loki/operator/internal/manifests/internal/config" - "github.com/stretchr/testify/require" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestNewQueryFrontendDeployment_SelectorMatchesLabels(t *testing.T) { @@ -52,10 +45,11 @@ func TestNewQueryFrontendDeployment_HasTemplateConfigHashAnnotation(t *testing.T require.Equal(t, annotations[expected], "deadbeef") } -func TestConfigureQueryFrontendHTTPServicePKI(t *testing.T) { - opts := Options{ - Name: "abcd", - Namespace: "efgh", +func TestNewQueryFrontendDeployment_HasTemplateCertRotationRequiredAtAnnotation(t *testing.T) { + ss := NewQueryFrontendDeployment(Options{ + Name: "abcd", + Namespace: "efgh", + CertRotationRequiredAt: "deadbeef", Stack: lokiv1.LokiStackSpec{ Template: &lokiv1.LokiTemplateSpec{ QueryFrontend: &lokiv1.LokiComponentSpec{ @@ -63,144 +57,10 @@ func TestConfigureQueryFrontendHTTPServicePKI(t *testing.T) { }, }, }, - TLSProfile: TLSProfileSpec{ - MinTLSVersion: "TLSVersion1.2", - Ciphers: []string{"TLS_RSA_WITH_AES_128_CBC_SHA"}, - }, - } - d := appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{ - Kind: "Deployment", - APIVersion: appsv1.SchemeGroupVersion.String(), - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: lokiFrontendContainerName, - Args: []string{ - "-target=query-frontend", - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: configVolumeName, - ReadOnly: false, - MountPath: config.LokiConfigMountDir, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: configVolumeName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - DefaultMode: &defaultConfigMapMode, - LocalObjectReference: corev1.LocalObjectReference{ - Name: lokiConfigMapName(opts.Name), - }, - }, - }, - }, - }, - }, - }, - }, - } - - caBundleVolumeName := signingCABundleName(opts.Name) - serviceName := serviceNameQueryFrontendHTTP(opts.Name) - expected := appsv1.Deployment{ - TypeMeta: metav1.TypeMeta{ - Kind: "Deployment", - APIVersion: appsv1.SchemeGroupVersion.String(), - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: lokiFrontendContainerName, - Args: []string{ - "-target=query-frontend", - "-frontend.tail-tls-config.tls-cipher-suites=TLS_RSA_WITH_AES_128_CBC_SHA", - "-frontend.tail-tls-config.tls-min-version=TLSVersion1.2", - fmt.Sprintf("-frontend.tail-tls-config.tls-ca-path=%s/%s", caBundleDir, caFile), - fmt.Sprintf("-server.http-tls-cert-path=%s", path.Join(httpTLSDir, tlsCertFile)), - fmt.Sprintf("-server.http-tls-key-path=%s", path.Join(httpTLSDir, tlsKeyFile)), - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: configVolumeName, - ReadOnly: false, - MountPath: config.LokiConfigMountDir, - }, - { - Name: caBundleVolumeName, - ReadOnly: true, - MountPath: caBundleDir, - }, - { - Name: serviceName, - ReadOnly: false, - MountPath: httpTLSDir, - }, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: corev1.ProbeHandler{ - HTTPGet: &corev1.HTTPGetAction{ - Scheme: corev1.URISchemeHTTPS, - }, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: configVolumeName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - DefaultMode: &defaultConfigMapMode, - LocalObjectReference: corev1.LocalObjectReference{ - Name: lokiConfigMapName(opts.Name), - }, - }, - }, - }, - { - Name: caBundleVolumeName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - DefaultMode: &defaultConfigMapMode, - LocalObjectReference: corev1.LocalObjectReference{ - Name: caBundleVolumeName, - }, - }, - }, - }, - { - Name: serviceName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: serviceName, - }, - }, - }, - }, - }, - }, - }, - } + }) - err := configureQueryFrontendHTTPServicePKI(&d, opts) - require.Nil(t, err) - require.Equal(t, expected, d) + expected := "loki.grafana.com/certRotationRequiredAt" + annotations := ss.Spec.Template.Annotations + require.Contains(t, annotations, expected) + require.Equal(t, annotations[expected], "deadbeef") } diff --git a/operator/internal/manifests/ruler.go b/operator/internal/manifests/ruler.go index e8537279f54e..6bdf510348b3 100644 --- a/operator/internal/manifests/ruler.go +++ b/operator/internal/manifests/ruler.go @@ -35,8 +35,14 @@ func BuildRuler(opts Options) ([]client.Object, error) { } } - objs := []client.Object{} + if opts.Gates.HTTPEncryption || opts.Gates.GRPCEncryption { + caBundleName := signingCABundleName(opts.Name) + if err := configureServiceCA(&statefulSet.Spec.Template.Spec, caBundleName); err != nil { + return nil, err + } + } + objs := []client.Object{} if opts.Stack.Tenants != nil { if err := configureRulerStatefulSetForMode(statefulSet, opts.Stack.Tenants.Mode, opts.Name); err != nil { return nil, err @@ -157,7 +163,7 @@ func NewRulerStatefulSet(opts Options) *appsv1.StatefulSet { } l := ComponentLabels(LabelRulerComponent, opts.Name) - a := commonAnnotations(opts.ConfigSHA1) + a := commonAnnotations(opts.ConfigSHA1, opts.CertRotationRequiredAt) return &appsv1.StatefulSet{ TypeMeta: metav1.TypeMeta{ @@ -230,43 +236,6 @@ func NewRulerStatefulSet(opts Options) *appsv1.StatefulSet { } } -func configureRulerStatefulSetForMode( - ss *appsv1.StatefulSet, mode lokiv1.ModeType, - stackName string, -) error { - switch mode { - case lokiv1.Static, lokiv1.Dynamic: - return nil // nothing to configure - case lokiv1.OpenshiftLogging, lokiv1.OpenshiftNetwork: - caBundleName := signingCABundleName(stackName) - monitorServerName := fqdn(openshift.MonitoringSVCMain, openshift.MonitoringNS) - return openshift.ConfigureRulerStatefulSet( - ss, - BearerTokenFile, - caBundleName, - caBundleDir, - caFile, - monitorServerName, - rulerContainerName, - ) - } - - return nil -} - -func configureRulerObjsForMode(opts Options) []client.Object { - openShiftObjs := []client.Object{} - - switch opts.Stack.Tenants.Mode { - case lokiv1.Static, lokiv1.Dynamic: - // nothing to configure - case lokiv1.OpenshiftLogging, lokiv1.OpenshiftNetwork: - openShiftObjs = openshift.BuildRulerObjects(opts.OpenShiftOptions) - } - - return openShiftObjs -} - // NewRulerGRPCService creates a k8s service for the ruler GRPC endpoint func NewRulerGRPCService(opts Options) *corev1.Service { serviceName := serviceNameRulerGRPC(opts.Name) @@ -278,9 +247,8 @@ func NewRulerGRPCService(opts Options) *corev1.Service { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: labels, - Annotations: serviceAnnotations(serviceName, opts.Gates.OpenShift.ServingCertsService), + Name: serviceName, + Labels: labels, }, Spec: corev1.ServiceSpec{ ClusterIP: "None", @@ -308,9 +276,8 @@ func NewRulerHTTPService(opts Options) *corev1.Service { APIVersion: corev1.SchemeGroupVersion.String(), }, ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - Labels: labels, - Annotations: serviceAnnotations(serviceName, opts.Gates.OpenShift.ServingCertsService), + Name: serviceName, + Labels: labels, }, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ @@ -328,60 +295,39 @@ func NewRulerHTTPService(opts Options) *corev1.Service { func configureRulerHTTPServicePKI(statefulSet *appsv1.StatefulSet, opts Options) error { serviceName := serviceNameRulerHTTP(opts.Name) - return configureHTTPServicePKI(&statefulSet.Spec.Template.Spec, serviceName) + return configureHTTPServicePKI(&statefulSet.Spec.Template.Spec, serviceName, opts.TLSProfile.MinTLSVersion, opts.TLSCipherSuites()) } func configureRulerGRPCServicePKI(sts *appsv1.StatefulSet, opts Options) error { - caBundleName := signingCABundleName(opts.Name) - secretVolumeSpec := corev1.PodSpec{ - Volumes: []corev1.Volume{ - { - Name: caBundleName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: caBundleName, - }, - }, - }, - }, - }, - } - secretContainerSpec := corev1.Container{ - VolumeMounts: []corev1.VolumeMount{ - { - Name: caBundleName, - ReadOnly: false, - MountPath: caBundleDir, - }, - }, Args: []string{ - // Enable GRPC over TLS for ruler client - "-ruler.client.tls-enabled=true", - fmt.Sprintf("-ruler.client.tls-cipher-suites=%s", opts.TLSCipherSuites()), - fmt.Sprintf("-ruler.client.tls-min-version=%s", opts.TLSProfile.MinTLSVersion), - fmt.Sprintf("-ruler.client.tls-ca-path=%s", signingCAPath()), - fmt.Sprintf("-ruler.client.tls-server-name=%s", fqdn(serviceNameRulerGRPC(opts.Name), opts.Namespace)), - // Enable GRPC over TLS for ingester client - "-ingester.client.tls-enabled=true", - fmt.Sprintf("-ingester.client.tls-cipher-suites=%s", opts.TLSCipherSuites()), - fmt.Sprintf("-ingester.client.tls-min-version=%s", opts.TLSProfile.MinTLSVersion), - fmt.Sprintf("-ingester.client.tls-ca-path=%s", signingCAPath()), - fmt.Sprintf("-ingester.client.tls-server-name=%s", fqdn(serviceNameIngesterGRPC(opts.Name), opts.Namespace)), // Enable GRPC over TLS for boltb-shipper index-gateway client "-boltdb.shipper.index-gateway-client.grpc.tls-enabled=true", fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-cipher-suites=%s", opts.TLSCipherSuites()), fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-min-version=%s", opts.TLSProfile.MinTLSVersion), fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-key-path=%s", lokiServerGRPCTLSKey()), fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-server-name=%s", fqdn(serviceNameIndexGatewayGRPC(opts.Name), opts.Namespace)), + // Enable GRPC over TLS for ingester client + "-ingester.client.tls-enabled=true", + fmt.Sprintf("-ingester.client.tls-cipher-suites=%s", opts.TLSCipherSuites()), + fmt.Sprintf("-ingester.client.tls-min-version=%s", opts.TLSProfile.MinTLSVersion), + fmt.Sprintf("-ingester.client.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-ingester.client.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-ingester.client.tls-key-path=%s", lokiServerGRPCTLSKey()), + fmt.Sprintf("-ingester.client.tls-server-name=%s", fqdn(serviceNameIngesterGRPC(opts.Name), opts.Namespace)), + // Enable GRPC over TLS for ruler client + "-ruler.client.tls-enabled=true", + fmt.Sprintf("-ruler.client.tls-cipher-suites=%s", opts.TLSCipherSuites()), + fmt.Sprintf("-ruler.client.tls-min-version=%s", opts.TLSProfile.MinTLSVersion), + fmt.Sprintf("-ruler.client.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-ruler.client.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-ruler.client.tls-key-path=%s", lokiServerGRPCTLSKey()), + fmt.Sprintf("-ruler.client.tls-server-name=%s", fqdn(serviceNameRulerGRPC(opts.Name), opts.Namespace)), }, } - if err := mergo.Merge(&sts.Spec.Template.Spec, secretVolumeSpec, mergo.WithAppendSlice); err != nil { - return kverrors.Wrap(err, "failed to merge volumes") - } - if err := mergo.Merge(&sts.Spec.Template.Spec.Containers[0], secretContainerSpec, mergo.WithAppendSlice); err != nil { return kverrors.Wrap(err, "failed to merge container") } @@ -390,6 +336,43 @@ func configureRulerGRPCServicePKI(sts *appsv1.StatefulSet, opts Options) error { return configureGRPCServicePKI(&sts.Spec.Template.Spec, serviceName) } +func configureRulerStatefulSetForMode( + ss *appsv1.StatefulSet, mode lokiv1.ModeType, + stackName string, +) error { + switch mode { + case lokiv1.Static, lokiv1.Dynamic: + return nil // nothing to configure + case lokiv1.OpenshiftLogging, lokiv1.OpenshiftNetwork: + bundleName := alertmanagerSigningCABundleName(ss.Name) + monitorServerName := fqdn(openshift.MonitoringSVCMain, openshift.MonitoringNS) + return openshift.ConfigureRulerStatefulSet( + ss, + bundleName, + BearerTokenFile, + alertmanagerUpstreamCADir(), + alertmanagerUpstreamCAPath(), + monitorServerName, + rulerContainerName, + ) + } + + return nil +} + +func configureRulerObjsForMode(opts Options) []client.Object { + openShiftObjs := []client.Object{} + + switch opts.Stack.Tenants.Mode { + case lokiv1.Static, lokiv1.Dynamic: + // nothing to configure + case lokiv1.OpenshiftLogging, lokiv1.OpenshiftNetwork: + openShiftObjs = openshift.BuildRulerObjects(opts.OpenShiftOptions) + } + + return openShiftObjs +} + func ruleVolumeItems(tenants map[string]TenantConfig) []corev1.KeyToPath { var items []corev1.KeyToPath diff --git a/operator/internal/manifests/ruler_test.go b/operator/internal/manifests/ruler_test.go index ff3591637540..2288ab046535 100644 --- a/operator/internal/manifests/ruler_test.go +++ b/operator/internal/manifests/ruler_test.go @@ -1,10 +1,12 @@ package manifests_test import ( + "math/rand" "testing" lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" "github.com/grafana/loki/operator/internal/manifests" + "github.com/grafana/loki/operator/internal/manifests/openshift" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" ) @@ -30,6 +32,55 @@ func TestNewRulerStatefulSet_HasTemplateConfigHashAnnotation(t *testing.T) { require.Equal(t, annotations[expected], "deadbeef") } +func TestNewRulerStatefulSet_HasTemplateCertRotationRequiredAtAnnotation(t *testing.T) { + ss := manifests.NewRulerStatefulSet(manifests.Options{ + Name: "abcd", + Namespace: "efgh", + CertRotationRequiredAt: "deadbeef", + Stack: lokiv1.LokiStackSpec{ + StorageClassName: "standard", + Template: &lokiv1.LokiTemplateSpec{ + Ruler: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + }) + expected := "loki.grafana.com/certRotationRequiredAt" + annotations := ss.Spec.Template.Annotations + require.Contains(t, annotations, expected) + require.Equal(t, annotations[expected], "deadbeef") +} + +func TestBuildRuler_HasExtraObjectsForTenantMode(t *testing.T) { + objs, err := manifests.BuildRuler(manifests.Options{ + Name: "abcd", + Namespace: "efgh", + OpenShiftOptions: openshift.Options{ + BuildOpts: openshift.BuildOptions{ + LokiStackName: "abc", + LokiStackNamespace: "efgh", + }, + }, + Stack: lokiv1.LokiStackSpec{ + Template: &lokiv1.LokiTemplateSpec{ + Ruler: &lokiv1.LokiComponentSpec{ + Replicas: rand.Int31(), + }, + }, + Rules: &lokiv1.RulesSpec{ + Enabled: true, + }, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.OpenshiftLogging, + }, + }, + }) + + require.NoError(t, err) + require.Len(t, objs, 7) +} + func TestNewRulerStatefulSet_SelectorMatchesLabels(t *testing.T) { // You must set the .spec.selector field of a StatefulSet to match the labels of // its .spec.template.metadata.labels. Prior to Kubernetes 1.8, the diff --git a/operator/internal/manifests/service.go b/operator/internal/manifests/service.go index 19ce5fea0554..8ed662345271 100644 --- a/operator/internal/manifests/service.go +++ b/operator/internal/manifests/service.go @@ -2,13 +2,50 @@ package manifests import ( "fmt" - "path" "github.com/ViaQ/logerr/v2/kverrors" "github.com/imdario/mergo" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) +func configureServiceCA(podSpec *corev1.PodSpec, caBundleName string) error { + secretVolumeSpec := corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: caBundleName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: caBundleName, + }, + }, + }, + }, + }, + } + + secretContainerSpec := corev1.Container{ + VolumeMounts: []corev1.VolumeMount{ + { + Name: caBundleName, + ReadOnly: false, + MountPath: caBundleDir, + }, + }, + } + + if err := mergo.Merge(podSpec, secretVolumeSpec, mergo.WithAppendSlice); err != nil { + return kverrors.Wrap(err, "failed to merge volumes") + } + + if err := mergo.Merge(&podSpec.Containers[0], secretContainerSpec, mergo.WithAppendSlice); err != nil { + return kverrors.Wrap(err, "failed to merge container") + } + + return nil +} + func configureGRPCServicePKI(podSpec *corev1.PodSpec, serviceName string) error { secretVolumeSpec := corev1.PodSpec{ Volumes: []corev1.Volume{ @@ -27,12 +64,14 @@ func configureGRPCServicePKI(podSpec *corev1.PodSpec, serviceName string) error { Name: serviceName, ReadOnly: false, - MountPath: grpcTLSDir, + MountPath: lokiServerGRPCTLSDir(), }, }, Args: []string{ - fmt.Sprintf("-server.grpc-tls-cert-path=%s", path.Join(grpcTLSDir, tlsCertFile)), - fmt.Sprintf("-server.grpc-tls-key-path=%s", path.Join(grpcTLSDir, tlsKeyFile)), + fmt.Sprintf("-server.grpc-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.grpc-tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-server.grpc-tls-key-path=%s", lokiServerGRPCTLSKey()), + "-server.grpc-tls-client-auth=RequireAndVerifyClientCert", }, } @@ -47,7 +86,7 @@ func configureGRPCServicePKI(podSpec *corev1.PodSpec, serviceName string) error return nil } -func configureHTTPServicePKI(podSpec *corev1.PodSpec, serviceName string) error { +func configureHTTPServicePKI(podSpec *corev1.PodSpec, serviceName, minTLSVersion, tlsCipherSuites string) error { secretVolumeSpec := corev1.PodSpec{ Volumes: []corev1.Volume{ { @@ -60,24 +99,44 @@ func configureHTTPServicePKI(podSpec *corev1.PodSpec, serviceName string) error }, }, } + secretContainerSpec := corev1.Container{ VolumeMounts: []corev1.VolumeMount{ { Name: serviceName, ReadOnly: false, - MountPath: httpTLSDir, + MountPath: lokiServerHTTPTLSDir(), }, }, Args: []string{ - fmt.Sprintf("-server.http-tls-cert-path=%s", path.Join(httpTLSDir, tlsCertFile)), - fmt.Sprintf("-server.http-tls-key-path=%s", path.Join(httpTLSDir, tlsKeyFile)), + // Expose ready handler through internal server without requiring mTLS + "-internal-server.enable=true", + "-internal-server.http-listen-address=", + fmt.Sprintf("-internal-server.http-tls-min-version=%s", minTLSVersion), + fmt.Sprintf("-internal-server.http-tls-cipher-suites=%s", tlsCipherSuites), + fmt.Sprintf("-internal-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-internal-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + // Require mTLS for any other handler + fmt.Sprintf("-server.http-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-server.http-tls-client-auth=RequireAndVerifyClientCert", + }, + Ports: []corev1.ContainerPort{ + { + Name: lokiInternalHTTPPortName, + ContainerPort: internalHTTPPort, + Protocol: protocolTCP, + }, }, } + uriSchemeContainerSpec := corev1.Container{ ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Scheme: corev1.URISchemeHTTPS, + Port: intstr.FromInt(internalHTTPPort), }, }, }, @@ -85,6 +144,7 @@ func configureHTTPServicePKI(podSpec *corev1.PodSpec, serviceName string) error ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ Scheme: corev1.URISchemeHTTPS, + Port: intstr.FromInt(internalHTTPPort), }, }, }, diff --git a/operator/internal/manifests/service_monitor.go b/operator/internal/manifests/service_monitor.go index bf8b62a90ba5..9d9b3636cc6c 100644 --- a/operator/internal/manifests/service_monitor.go +++ b/operator/internal/manifests/service_monitor.go @@ -28,7 +28,7 @@ func NewDistributorServiceMonitor(opts Options) *monitoringv1.ServiceMonitor { serviceMonitorName := serviceMonitorName(DistributorName(opts.Name)) serviceName := serviceNameDistributorHTTP(opts.Name) - lokiEndpoint := serviceMonitorEndpoint(lokiHTTPPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) + lokiEndpoint := lokiServiceMonitorEndpoint(opts.Name, lokiHTTPPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) return newServiceMonitor(opts.Namespace, serviceMonitorName, l, lokiEndpoint) } @@ -39,7 +39,7 @@ func NewIngesterServiceMonitor(opts Options) *monitoringv1.ServiceMonitor { serviceMonitorName := serviceMonitorName(IngesterName(opts.Name)) serviceName := serviceNameIngesterHTTP(opts.Name) - lokiEndpoint := serviceMonitorEndpoint(lokiHTTPPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) + lokiEndpoint := lokiServiceMonitorEndpoint(opts.Name, lokiHTTPPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) return newServiceMonitor(opts.Namespace, serviceMonitorName, l, lokiEndpoint) } @@ -50,7 +50,7 @@ func NewQuerierServiceMonitor(opts Options) *monitoringv1.ServiceMonitor { serviceMonitorName := serviceMonitorName(QuerierName(opts.Name)) serviceName := serviceNameQuerierHTTP(opts.Name) - lokiEndpoint := serviceMonitorEndpoint(lokiHTTPPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) + lokiEndpoint := lokiServiceMonitorEndpoint(opts.Name, lokiHTTPPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) return newServiceMonitor(opts.Namespace, serviceMonitorName, l, lokiEndpoint) } @@ -61,7 +61,7 @@ func NewCompactorServiceMonitor(opts Options) *monitoringv1.ServiceMonitor { serviceMonitorName := serviceMonitorName(CompactorName(opts.Name)) serviceName := serviceNameCompactorHTTP(opts.Name) - lokiEndpoint := serviceMonitorEndpoint(lokiHTTPPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) + lokiEndpoint := lokiServiceMonitorEndpoint(opts.Name, lokiHTTPPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) return newServiceMonitor(opts.Namespace, serviceMonitorName, l, lokiEndpoint) } @@ -72,7 +72,7 @@ func NewQueryFrontendServiceMonitor(opts Options) *monitoringv1.ServiceMonitor { serviceMonitorName := serviceMonitorName(QueryFrontendName(opts.Name)) serviceName := serviceNameQueryFrontendHTTP(opts.Name) - lokiEndpoint := serviceMonitorEndpoint(lokiHTTPPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) + lokiEndpoint := lokiServiceMonitorEndpoint(opts.Name, lokiHTTPPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) return newServiceMonitor(opts.Namespace, serviceMonitorName, l, lokiEndpoint) } @@ -83,7 +83,7 @@ func NewIndexGatewayServiceMonitor(opts Options) *monitoringv1.ServiceMonitor { serviceMonitorName := serviceMonitorName(IndexGatewayName(opts.Name)) serviceName := serviceNameIndexGatewayHTTP(opts.Name) - lokiEndpoint := serviceMonitorEndpoint(lokiHTTPPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) + lokiEndpoint := lokiServiceMonitorEndpoint(opts.Name, lokiHTTPPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) return newServiceMonitor(opts.Namespace, serviceMonitorName, l, lokiEndpoint) } @@ -94,7 +94,7 @@ func NewRulerServiceMonitor(opts Options) *monitoringv1.ServiceMonitor { serviceMonitorName := serviceMonitorName(RulerName(opts.Name)) serviceName := serviceNameRulerHTTP(opts.Name) - lokiEndpoint := serviceMonitorEndpoint(lokiHTTPPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) + lokiEndpoint := lokiServiceMonitorEndpoint(opts.Name, lokiHTTPPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) return newServiceMonitor(opts.Namespace, serviceMonitorName, l, lokiEndpoint) } @@ -103,14 +103,15 @@ func NewRulerServiceMonitor(opts Options) *monitoringv1.ServiceMonitor { func NewGatewayServiceMonitor(opts Options) *monitoringv1.ServiceMonitor { l := ComponentLabels(LabelGatewayComponent, opts.Name) - serviceMonitorName := serviceMonitorName(GatewayName(opts.Name)) + gatewayName := GatewayName(opts.Name) + serviceMonitorName := serviceMonitorName(gatewayName) serviceName := serviceNameGatewayHTTP(opts.Name) - gwEndpoint := serviceMonitorEndpoint(gatewayInternalPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) + gwEndpoint := gatewayServiceMonitorEndpoint(gatewayName, gatewayInternalPortName, serviceName, opts.Namespace, opts.Gates.ServiceMonitorTLSEndpoints) sm := newServiceMonitor(opts.Namespace, serviceMonitorName, l, gwEndpoint) if opts.Stack.Tenants != nil { - if err := configureGatewayServiceMonitorForMode(sm, opts.Stack.Tenants.Mode, opts.Gates); err != nil { + if err := configureGatewayServiceMonitorForMode(sm, opts); err != nil { return sm } } diff --git a/operator/internal/manifests/service_monitor_test.go b/operator/internal/manifests/service_monitor_test.go index 34661dce10aa..de46dfbda14f 100644 --- a/operator/internal/manifests/service_monitor_test.go +++ b/operator/internal/manifests/service_monitor_test.go @@ -22,11 +22,9 @@ func TestServiceMonitorMatchLabels(t *testing.T) { } featureGates := configv1.FeatureGates{ + BuiltInCertManagement: configv1.BuiltInCertManagement{Enabled: true}, ServiceMonitors: true, ServiceMonitorTLSEndpoints: true, - OpenShift: configv1.OpenShiftFeatureGates{ - ServingCertsService: true, - }, } opt := Options{ @@ -114,14 +112,16 @@ func TestServiceMonitorMatchLabels(t *testing.T) { } } -func TestServiceMonitorEndpoints_ForOpenShiftLoggingMode(t *testing.T) { +func TestServiceMonitorEndpoints_ForBuiltInCertRotation(t *testing.T) { + type test struct { + Service *corev1.Service + ServiceMonitor *monitoringv1.ServiceMonitor + } + featureGates := configv1.FeatureGates{ - LokiStackGateway: true, + BuiltInCertManagement: configv1.BuiltInCertManagement{Enabled: true}, ServiceMonitors: true, ServiceMonitorTLSEndpoints: true, - OpenShift: configv1.OpenShiftFeatureGates{ - ServingCertsService: true, - }, } opt := Options{ @@ -131,17 +131,398 @@ func TestServiceMonitorEndpoints_ForOpenShiftLoggingMode(t *testing.T) { Gates: featureGates, Stack: lokiv1.LokiStackSpec{ Size: lokiv1.SizeOneXExtraSmall, - Tenants: &lokiv1.TenantsSpec{ - Mode: lokiv1.OpenshiftLogging, - }, Template: &lokiv1.LokiTemplateSpec{ - Gateway: &lokiv1.LokiComponentSpec{ + Compactor: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + Distributor: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + Ingester: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + Querier: &lokiv1.LokiComponentSpec{ Replicas: 1, }, + QueryFrontend: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + IndexGateway: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + Ruler: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + } + + table := []test{ + { + Service: NewDistributorHTTPService(opt), + ServiceMonitor: NewDistributorServiceMonitor(opt), + }, + { + Service: NewIngesterHTTPService(opt), + ServiceMonitor: NewIngesterServiceMonitor(opt), + }, + { + Service: NewQuerierHTTPService(opt), + ServiceMonitor: NewQuerierServiceMonitor(opt), + }, + { + Service: NewQueryFrontendHTTPService(opt), + ServiceMonitor: NewQueryFrontendServiceMonitor(opt), + }, + { + Service: NewCompactorHTTPService(opt), + ServiceMonitor: NewCompactorServiceMonitor(opt), + }, + { + Service: NewIndexGatewayHTTPService(opt), + ServiceMonitor: NewIndexGatewayServiceMonitor(opt), + }, + { + Service: NewRulerHTTPService(opt), + ServiceMonitor: NewRulerServiceMonitor(opt), + }, + } + + for _, tst := range table { + testName := fmt.Sprintf("%s_%s", tst.Service.GetName(), tst.ServiceMonitor.GetName()) + t.Run(testName, func(t *testing.T) { + t.Parallel() + + require.NotNil(t, tst.ServiceMonitor.Spec.Endpoints) + require.NotNil(t, tst.ServiceMonitor.Spec.Endpoints[0].TLSConfig) + + // Do not use bearer authentication for loki endpoints + require.Empty(t, tst.ServiceMonitor.Spec.Endpoints[0].BearerTokenFile) + require.Empty(t, tst.ServiceMonitor.Spec.Endpoints[0].BearerTokenSecret) + + // Check using built-in PKI + c := tst.ServiceMonitor.Spec.Endpoints[0].TLSConfig + require.Equal(t, c.CA.ConfigMap.LocalObjectReference.Name, signingCABundleName(opt.Name)) + require.Equal(t, c.Cert.Secret.LocalObjectReference.Name, tst.Service.Name) + require.Equal(t, c.KeySecret.LocalObjectReference.Name, tst.Service.Name) + }) + } +} + +func TestServiceMonitorEndpoints_ForGatewayServiceMonitor(t *testing.T) { + tt := []struct { + desc string + opts Options + total int + want []monitoringv1.Endpoint + }{ + { + desc: "default", + opts: Options{ + Name: "test", + Namespace: "test", + Image: "test", + Stack: lokiv1.LokiStackSpec{ + Size: lokiv1.SizeOneXExtraSmall, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.Static, + }, + Template: &lokiv1.LokiTemplateSpec{ + Gateway: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + }, + total: 1, + want: []monitoringv1.Endpoint{ + { + Port: gatewayInternalPortName, + Path: "/metrics", + Scheme: "http", + }, + }, + }, + { + desc: "with http encryption", + opts: Options{ + Name: "test", + Namespace: "test", + Image: "test", + Gates: configv1.FeatureGates{ + LokiStackGateway: true, + BuiltInCertManagement: configv1.BuiltInCertManagement{Enabled: true}, + ServiceMonitors: true, + ServiceMonitorTLSEndpoints: true, + }, + Stack: lokiv1.LokiStackSpec{ + Size: lokiv1.SizeOneXExtraSmall, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.Static, + }, + Template: &lokiv1.LokiTemplateSpec{ + Gateway: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + }, + total: 1, + want: []monitoringv1.Endpoint{ + { + Port: gatewayInternalPortName, + Path: "/metrics", + Scheme: "https", + BearerTokenSecret: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test-gateway-token", + }, + Key: corev1.ServiceAccountTokenKey, + }, + TLSConfig: &monitoringv1.TLSConfig{ + SafeTLSConfig: monitoringv1.SafeTLSConfig{ + CA: monitoringv1.SecretOrConfigMap{ + ConfigMap: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: gatewaySigningCABundleName("test-gateway"), + }, + Key: caFile, + }, + }, + ServerName: "test-gateway-http.test.svc.cluster.local", + }, + }, + }, + }, + }, + { + desc: "openshift-logging", + opts: Options{ + Name: "test", + Namespace: "test", + Image: "test", + Stack: lokiv1.LokiStackSpec{ + Size: lokiv1.SizeOneXExtraSmall, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.OpenshiftLogging, + }, + Template: &lokiv1.LokiTemplateSpec{ + Gateway: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + }, + total: 2, + want: []monitoringv1.Endpoint{ + { + Port: gatewayInternalPortName, + Path: "/metrics", + Scheme: "http", + }, + { + Port: "opa-metrics", + Path: "/metrics", + Scheme: "http", + }, + }, + }, + { + desc: "openshift-logging with http encryption", + opts: Options{ + Name: "test", + Namespace: "test", + Image: "test", + Gates: configv1.FeatureGates{ + LokiStackGateway: true, + BuiltInCertManagement: configv1.BuiltInCertManagement{Enabled: true}, + ServiceMonitors: true, + ServiceMonitorTLSEndpoints: true, + }, + Stack: lokiv1.LokiStackSpec{ + Size: lokiv1.SizeOneXExtraSmall, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.OpenshiftLogging, + }, + Template: &lokiv1.LokiTemplateSpec{ + Gateway: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + }, + total: 2, + want: []monitoringv1.Endpoint{ + { + Port: gatewayInternalPortName, + Path: "/metrics", + Scheme: "https", + BearerTokenSecret: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test-gateway-token", + }, + Key: corev1.ServiceAccountTokenKey, + }, + TLSConfig: &monitoringv1.TLSConfig{ + SafeTLSConfig: monitoringv1.SafeTLSConfig{ + CA: monitoringv1.SecretOrConfigMap{ + ConfigMap: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: gatewaySigningCABundleName("test-gateway"), + }, + Key: caFile, + }, + }, + ServerName: "test-gateway-http.test.svc.cluster.local", + }, + }, + }, + { + Port: "opa-metrics", + Path: "/metrics", + Scheme: "https", + BearerTokenSecret: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test-gateway-token", + }, + Key: corev1.ServiceAccountTokenKey, + }, + TLSConfig: &monitoringv1.TLSConfig{ + SafeTLSConfig: monitoringv1.SafeTLSConfig{ + CA: monitoringv1.SecretOrConfigMap{ + ConfigMap: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: gatewaySigningCABundleName("test-gateway"), + }, + Key: caFile, + }, + }, + ServerName: "test-gateway-http.test.svc.cluster.local", + }, + }, + }, + }, + }, + { + desc: "openshift-network", + opts: Options{ + Name: "test", + Namespace: "test", + Image: "test", + Stack: lokiv1.LokiStackSpec{ + Size: lokiv1.SizeOneXExtraSmall, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.OpenshiftNetwork, + }, + Template: &lokiv1.LokiTemplateSpec{ + Gateway: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + }, + total: 2, + want: []monitoringv1.Endpoint{ + { + Port: gatewayInternalPortName, + Path: "/metrics", + Scheme: "http", + }, + { + Port: "opa-metrics", + Path: "/metrics", + Scheme: "http", + }, + }, + }, + { + desc: "openshift-network with http encryption", + opts: Options{ + Name: "test", + Namespace: "test", + Image: "test", + Gates: configv1.FeatureGates{ + LokiStackGateway: true, + BuiltInCertManagement: configv1.BuiltInCertManagement{Enabled: true}, + ServiceMonitors: true, + ServiceMonitorTLSEndpoints: true, + }, + Stack: lokiv1.LokiStackSpec{ + Size: lokiv1.SizeOneXExtraSmall, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.OpenshiftNetwork, + }, + Template: &lokiv1.LokiTemplateSpec{ + Gateway: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + }, + total: 2, + want: []monitoringv1.Endpoint{ + { + Port: gatewayInternalPortName, + Path: "/metrics", + Scheme: "https", + BearerTokenSecret: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test-gateway-token", + }, + Key: corev1.ServiceAccountTokenKey, + }, + TLSConfig: &monitoringv1.TLSConfig{ + SafeTLSConfig: monitoringv1.SafeTLSConfig{ + CA: monitoringv1.SecretOrConfigMap{ + ConfigMap: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: gatewaySigningCABundleName("test-gateway"), + }, + Key: caFile, + }, + }, + ServerName: "test-gateway-http.test.svc.cluster.local", + }, + }, + }, + { + Port: "opa-metrics", + Path: "/metrics", + Scheme: "https", + BearerTokenSecret: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "test-gateway-token", + }, + Key: corev1.ServiceAccountTokenKey, + }, + TLSConfig: &monitoringv1.TLSConfig{ + SafeTLSConfig: monitoringv1.SafeTLSConfig{ + CA: monitoringv1.SecretOrConfigMap{ + ConfigMap: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: gatewaySigningCABundleName("test-gateway"), + }, + Key: caFile, + }, + }, + ServerName: "test-gateway-http.test.svc.cluster.local", + }, + }, + }, }, }, } - sm := NewGatewayServiceMonitor(opt) - require.Len(t, sm.Spec.Endpoints, 2) + for _, tc := range tt { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + sm := NewGatewayServiceMonitor(tc.opts) + require.Len(t, sm.Spec.Endpoints, tc.total) + + for _, endpoint := range tc.want { + require.Contains(t, sm.Spec.Endpoints, endpoint) + } + }) + } } diff --git a/operator/internal/manifests/service_test.go b/operator/internal/manifests/service_test.go index 41a197650ba7..4f935b5bfd5d 100644 --- a/operator/internal/manifests/service_test.go +++ b/operator/internal/manifests/service_test.go @@ -2,10 +2,14 @@ package manifests import ( "fmt" + "strings" "testing" + configv1 "github.com/grafana/loki/operator/apis/config/v1" lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -257,3 +261,710 @@ func TestServicesMatchLabels(t *testing.T) { } } } + +func TestServices_WithEncryption(t *testing.T) { + const ( + stackName = "test" + stackNs = "ns" + ) + + opts := Options{ + Name: stackName, + Namespace: stackNs, + Gates: configv1.FeatureGates{ + HTTPEncryption: true, + GRPCEncryption: true, + }, + Stack: lokiv1.LokiStackSpec{ + Size: lokiv1.SizeOneXExtraSmall, + Template: &lokiv1.LokiTemplateSpec{ + Compactor: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + Distributor: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + Ingester: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + Querier: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + QueryFrontend: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + Gateway: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + IndexGateway: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + Ruler: &lokiv1.LokiComponentSpec{ + Replicas: 1, + }, + }, + }, + TLSProfile: TLSProfileSpec{ + MinTLSVersion: "VersionTLS12", + Ciphers: []string{"cipher1", "cipher2"}, + }, + } + + tt := []struct { + desc string + buildFunc func(Options) ([]client.Object, error) + wantArgs []string + wantPorts []corev1.ContainerPort + wantVolumeMounts []corev1.VolumeMount + wantVolumes []corev1.Volume + }{ + { + desc: "compactor", + buildFunc: BuildCompactor, + wantArgs: []string{ + "-internal-server.enable=true", + "-internal-server.http-listen-address=", + fmt.Sprintf("-internal-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-internal-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-internal-server.http-tls-cipher-suites=cipher1,cipher2", + "-internal-server.http-tls-min-version=VersionTLS12", + "-server.tls-cipher-suites=cipher1,cipher2", + "-server.tls-min-version=VersionTLS12", + fmt.Sprintf("-server.http-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-server.http-tls-client-auth=RequireAndVerifyClientCert", + fmt.Sprintf("-server.grpc-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.grpc-tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-server.grpc-tls-key-path=%s", lokiServerGRPCTLSKey()), + "-server.grpc-tls-client-auth=RequireAndVerifyClientCert", + }, + wantPorts: []corev1.ContainerPort{ + { + Name: lokiInternalHTTPPortName, + ContainerPort: internalHTTPPort, + Protocol: protocolTCP, + }, + }, + wantVolumeMounts: []corev1.VolumeMount{ + { + Name: serviceNameCompactorHTTP(stackName), + ReadOnly: false, + MountPath: lokiServerHTTPTLSDir(), + }, + { + Name: serviceNameCompactorGRPC(stackName), + ReadOnly: false, + MountPath: lokiServerGRPCTLSDir(), + }, + { + Name: signingCABundleName(stackName), + ReadOnly: false, + MountPath: caBundleDir, + }, + }, + wantVolumes: []corev1.Volume{ + { + Name: serviceNameCompactorHTTP(stackName), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceNameCompactorHTTP(stackName), + }, + }, + }, + { + Name: serviceNameCompactorGRPC(stackName), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceNameCompactorGRPC(stackName), + }, + }, + }, + { + Name: signingCABundleName(stackName), + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: signingCABundleName(stackName), + }, + }, + }, + }, + }, + }, + { + desc: "distributor", + buildFunc: BuildDistributor, + wantArgs: []string{ + "-ingester.client.tls-enabled=true", + fmt.Sprintf("-ingester.client.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-ingester.client.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-ingester.client.tls-key-path=%s", lokiServerGRPCTLSKey()), + fmt.Sprintf("-ingester.client.tls-server-name=%s", fqdn(serviceNameIngesterGRPC(stackName), stackNs)), + "-ingester.client.tls-min-version=VersionTLS12", + "-ingester.client.tls-cipher-suites=cipher1,cipher2", + "-internal-server.enable=true", + "-internal-server.http-listen-address=", + fmt.Sprintf("-internal-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-internal-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-internal-server.http-tls-cipher-suites=cipher1,cipher2", + "-internal-server.http-tls-min-version=VersionTLS12", + "-server.tls-cipher-suites=cipher1,cipher2", + "-server.tls-min-version=VersionTLS12", + fmt.Sprintf("-server.http-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-server.http-tls-client-auth=RequireAndVerifyClientCert", + fmt.Sprintf("-server.grpc-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.grpc-tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-server.grpc-tls-key-path=%s", lokiServerGRPCTLSKey()), + "-server.grpc-tls-client-auth=RequireAndVerifyClientCert", + }, + wantPorts: []corev1.ContainerPort{ + { + Name: lokiInternalHTTPPortName, + ContainerPort: internalHTTPPort, + Protocol: protocolTCP, + }, + }, + wantVolumeMounts: []corev1.VolumeMount{ + { + Name: serviceNameDistributorHTTP(stackName), + ReadOnly: false, + MountPath: lokiServerHTTPTLSDir(), + }, + { + Name: serviceNameDistributorGRPC(stackName), + ReadOnly: false, + MountPath: lokiServerGRPCTLSDir(), + }, + { + Name: signingCABundleName(stackName), + ReadOnly: false, + MountPath: caBundleDir, + }, + }, + wantVolumes: []corev1.Volume{ + { + Name: serviceNameDistributorHTTP(stackName), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceNameDistributorHTTP(stackName), + }, + }, + }, + { + Name: serviceNameDistributorGRPC(stackName), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceNameDistributorGRPC(stackName), + }, + }, + }, + { + Name: signingCABundleName(stackName), + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: signingCABundleName(stackName), + }, + }, + }, + }, + }, + }, + { + desc: "index-gateway", + buildFunc: BuildIndexGateway, + wantArgs: []string{ + "-internal-server.enable=true", + "-internal-server.http-listen-address=", + fmt.Sprintf("-internal-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-internal-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-internal-server.http-tls-cipher-suites=cipher1,cipher2", + "-internal-server.http-tls-min-version=VersionTLS12", + "-server.tls-cipher-suites=cipher1,cipher2", + "-server.tls-min-version=VersionTLS12", + fmt.Sprintf("-server.http-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-server.http-tls-client-auth=RequireAndVerifyClientCert", + fmt.Sprintf("-server.grpc-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.grpc-tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-server.grpc-tls-key-path=%s", lokiServerGRPCTLSKey()), + "-server.grpc-tls-client-auth=RequireAndVerifyClientCert", + }, + wantPorts: []corev1.ContainerPort{ + { + Name: lokiInternalHTTPPortName, + ContainerPort: internalHTTPPort, + Protocol: protocolTCP, + }, + }, + wantVolumeMounts: []corev1.VolumeMount{ + { + Name: serviceNameIndexGatewayHTTP(stackName), + ReadOnly: false, + MountPath: lokiServerHTTPTLSDir(), + }, + { + Name: serviceNameIndexGatewayGRPC(stackName), + ReadOnly: false, + MountPath: lokiServerGRPCTLSDir(), + }, + { + Name: signingCABundleName(stackName), + ReadOnly: false, + MountPath: caBundleDir, + }, + }, + wantVolumes: []corev1.Volume{ + { + Name: serviceNameIndexGatewayHTTP(stackName), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceNameIndexGatewayHTTP(stackName), + }, + }, + }, + { + Name: serviceNameIndexGatewayGRPC(stackName), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceNameIndexGatewayGRPC(stackName), + }, + }, + }, + { + Name: signingCABundleName(stackName), + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: signingCABundleName(stackName), + }, + }, + }, + }, + }, + }, + { + desc: "ingester", + buildFunc: BuildIngester, + wantArgs: []string{ + "-ingester.client.tls-enabled=true", + fmt.Sprintf("-ingester.client.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-ingester.client.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-ingester.client.tls-key-path=%s", lokiServerGRPCTLSKey()), + fmt.Sprintf("-ingester.client.tls-server-name=%s", fqdn(serviceNameIngesterGRPC(stackName), stackNs)), + "-ingester.client.tls-min-version=VersionTLS12", + "-ingester.client.tls-cipher-suites=cipher1,cipher2", + "-boltdb.shipper.index-gateway-client.grpc.tls-enabled=true", + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-key-path=%s", lokiServerGRPCTLSKey()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-server-name=%s", fqdn(serviceNameIndexGatewayGRPC(stackName), stackNs)), + "-boltdb.shipper.index-gateway-client.grpc.tls-min-version=VersionTLS12", + "-boltdb.shipper.index-gateway-client.grpc.tls-cipher-suites=cipher1,cipher2", + "-internal-server.enable=true", + "-internal-server.http-listen-address=", + fmt.Sprintf("-internal-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-internal-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-internal-server.http-tls-cipher-suites=cipher1,cipher2", + "-internal-server.http-tls-min-version=VersionTLS12", + "-server.tls-cipher-suites=cipher1,cipher2", + "-server.tls-min-version=VersionTLS12", + fmt.Sprintf("-server.http-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-server.http-tls-client-auth=RequireAndVerifyClientCert", + fmt.Sprintf("-server.grpc-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.grpc-tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-server.grpc-tls-key-path=%s", lokiServerGRPCTLSKey()), + "-server.grpc-tls-client-auth=RequireAndVerifyClientCert", + }, + wantPorts: []corev1.ContainerPort{ + { + Name: lokiInternalHTTPPortName, + ContainerPort: internalHTTPPort, + Protocol: protocolTCP, + }, + }, + wantVolumeMounts: []corev1.VolumeMount{ + { + Name: serviceNameIngesterHTTP(stackName), + ReadOnly: false, + MountPath: lokiServerHTTPTLSDir(), + }, + { + Name: serviceNameIngesterGRPC(stackName), + ReadOnly: false, + MountPath: lokiServerGRPCTLSDir(), + }, + { + Name: signingCABundleName(stackName), + ReadOnly: false, + MountPath: caBundleDir, + }, + }, + wantVolumes: []corev1.Volume{ + { + Name: serviceNameIngesterHTTP(stackName), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceNameIngesterHTTP(stackName), + }, + }, + }, + { + Name: serviceNameIngesterGRPC(stackName), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceNameIngesterGRPC(stackName), + }, + }, + }, + { + Name: signingCABundleName(stackName), + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: signingCABundleName(stackName), + }, + }, + }, + }, + }, + }, + { + desc: "querier", + buildFunc: BuildQuerier, + wantArgs: []string{ + "-ingester.client.tls-enabled=true", + fmt.Sprintf("-ingester.client.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-ingester.client.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-ingester.client.tls-key-path=%s", lokiServerGRPCTLSKey()), + fmt.Sprintf("-ingester.client.tls-server-name=%s", fqdn(serviceNameIngesterGRPC(stackName), stackNs)), + "-ingester.client.tls-min-version=VersionTLS12", + "-ingester.client.tls-cipher-suites=cipher1,cipher2", + "-querier.frontend-client.tls-enabled=true", + fmt.Sprintf("-querier.frontend-client.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-querier.frontend-client.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-querier.frontend-client.tls-key-path=%s", lokiServerGRPCTLSKey()), + fmt.Sprintf("-querier.frontend-client.tls-server-name=%s", fqdn(serviceNameQueryFrontendGRPC(stackName), stackNs)), + "-querier.frontend-client.tls-min-version=VersionTLS12", + "-querier.frontend-client.tls-cipher-suites=cipher1,cipher2", + "-boltdb.shipper.index-gateway-client.grpc.tls-enabled=true", + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-key-path=%s", lokiServerGRPCTLSKey()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-server-name=%s", fqdn(serviceNameIndexGatewayGRPC(stackName), stackNs)), + "-boltdb.shipper.index-gateway-client.grpc.tls-min-version=VersionTLS12", + "-boltdb.shipper.index-gateway-client.grpc.tls-cipher-suites=cipher1,cipher2", + "-internal-server.enable=true", + "-internal-server.http-listen-address=", + fmt.Sprintf("-internal-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-internal-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-internal-server.http-tls-cipher-suites=cipher1,cipher2", + "-internal-server.http-tls-min-version=VersionTLS12", + "-server.tls-cipher-suites=cipher1,cipher2", + "-server.tls-min-version=VersionTLS12", + fmt.Sprintf("-server.http-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-server.http-tls-client-auth=RequireAndVerifyClientCert", + fmt.Sprintf("-server.grpc-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.grpc-tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-server.grpc-tls-key-path=%s", lokiServerGRPCTLSKey()), + "-server.grpc-tls-client-auth=RequireAndVerifyClientCert", + }, + wantPorts: []corev1.ContainerPort{ + { + Name: lokiInternalHTTPPortName, + ContainerPort: internalHTTPPort, + Protocol: protocolTCP, + }, + }, + wantVolumeMounts: []corev1.VolumeMount{ + { + Name: serviceNameQuerierHTTP(stackName), + ReadOnly: false, + MountPath: lokiServerHTTPTLSDir(), + }, + { + Name: serviceNameQuerierGRPC(stackName), + ReadOnly: false, + MountPath: lokiServerGRPCTLSDir(), + }, + { + Name: signingCABundleName(stackName), + ReadOnly: false, + MountPath: caBundleDir, + }, + }, + wantVolumes: []corev1.Volume{ + { + Name: serviceNameQuerierHTTP(stackName), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceNameQuerierHTTP(stackName), + }, + }, + }, + { + Name: serviceNameQuerierGRPC(stackName), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceNameQuerierGRPC(stackName), + }, + }, + }, + { + Name: signingCABundleName(stackName), + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: signingCABundleName(stackName), + }, + }, + }, + }, + }, + }, + { + desc: "query-frontend", + buildFunc: BuildQueryFrontend, + wantArgs: []string{ + "-frontend.tail-tls-config.tls-min-version=VersionTLS12", + "-frontend.tail-tls-config.tls-cipher-suites=cipher1,cipher2", + fmt.Sprintf("-frontend.tail-tls-config.tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-frontend.tail-tls-config.tls-key-path=%s", lokiServerHTTPTLSKey()), + "-frontend.tail-proxy-url=https://test-querier-http.ns.svc.cluster.local:3100", + fmt.Sprintf("-frontend.tail-tls-config.tls-ca-path=%s", signingCAPath()), + "-internal-server.enable=true", + "-internal-server.http-listen-address=", + fmt.Sprintf("-internal-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-internal-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-internal-server.http-tls-cipher-suites=cipher1,cipher2", + "-internal-server.http-tls-min-version=VersionTLS12", + "-server.tls-cipher-suites=cipher1,cipher2", + "-server.tls-min-version=VersionTLS12", + fmt.Sprintf("-server.http-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-server.http-tls-client-auth=RequireAndVerifyClientCert", + fmt.Sprintf("-server.grpc-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.grpc-tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-server.grpc-tls-key-path=%s", lokiServerGRPCTLSKey()), + "-server.grpc-tls-client-auth=RequireAndVerifyClientCert", + }, + wantPorts: []corev1.ContainerPort{ + { + Name: lokiInternalHTTPPortName, + ContainerPort: internalHTTPPort, + Protocol: protocolTCP, + }, + }, + wantVolumeMounts: []corev1.VolumeMount{ + { + Name: serviceNameQueryFrontendHTTP(stackName), + ReadOnly: false, + MountPath: lokiServerHTTPTLSDir(), + }, + { + Name: serviceNameQueryFrontendGRPC(stackName), + ReadOnly: false, + MountPath: lokiServerGRPCTLSDir(), + }, + { + Name: signingCABundleName(stackName), + ReadOnly: false, + MountPath: caBundleDir, + }, + }, + wantVolumes: []corev1.Volume{ + { + Name: serviceNameQueryFrontendHTTP(stackName), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceNameQueryFrontendHTTP(stackName), + }, + }, + }, + { + Name: serviceNameQueryFrontendGRPC(stackName), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceNameQueryFrontendGRPC(stackName), + }, + }, + }, + { + Name: signingCABundleName(stackName), + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: signingCABundleName(stackName), + }, + }, + }, + }, + }, + }, + { + desc: "ruler", + buildFunc: BuildRuler, + wantArgs: []string{ + "-boltdb.shipper.index-gateway-client.grpc.tls-enabled=true", + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-key-path=%s", lokiServerGRPCTLSKey()), + fmt.Sprintf("-boltdb.shipper.index-gateway-client.grpc.tls-server-name=%s", fqdn(serviceNameIndexGatewayGRPC(stackName), stackNs)), + "-boltdb.shipper.index-gateway-client.grpc.tls-min-version=VersionTLS12", + "-boltdb.shipper.index-gateway-client.grpc.tls-cipher-suites=cipher1,cipher2", + "-ingester.client.tls-enabled=true", + fmt.Sprintf("-ingester.client.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-ingester.client.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-ingester.client.tls-key-path=%s", lokiServerGRPCTLSKey()), + fmt.Sprintf("-ingester.client.tls-server-name=%s", fqdn(serviceNameIngesterGRPC(stackName), stackNs)), + "-ingester.client.tls-min-version=VersionTLS12", + "-ingester.client.tls-cipher-suites=cipher1,cipher2", + "-ruler.client.tls-enabled=true", + fmt.Sprintf("-ruler.client.tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-ruler.client.tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-ruler.client.tls-key-path=%s", lokiServerGRPCTLSKey()), + fmt.Sprintf("-ruler.client.tls-server-name=%s", fqdn(serviceNameRulerGRPC(stackName), stackNs)), + "-ruler.client.tls-min-version=VersionTLS12", + "-ruler.client.tls-cipher-suites=cipher1,cipher2", + "-internal-server.enable=true", + "-internal-server.http-listen-address=", + fmt.Sprintf("-internal-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-internal-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-internal-server.http-tls-cipher-suites=cipher1,cipher2", + "-internal-server.http-tls-min-version=VersionTLS12", + "-server.tls-cipher-suites=cipher1,cipher2", + "-server.tls-min-version=VersionTLS12", + fmt.Sprintf("-server.http-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.http-tls-cert-path=%s", lokiServerHTTPTLSCert()), + fmt.Sprintf("-server.http-tls-key-path=%s", lokiServerHTTPTLSKey()), + "-server.http-tls-client-auth=RequireAndVerifyClientCert", + fmt.Sprintf("-server.grpc-tls-ca-path=%s", signingCAPath()), + fmt.Sprintf("-server.grpc-tls-cert-path=%s", lokiServerGRPCTLSCert()), + fmt.Sprintf("-server.grpc-tls-key-path=%s", lokiServerGRPCTLSKey()), + "-server.grpc-tls-client-auth=RequireAndVerifyClientCert", + }, + wantPorts: []corev1.ContainerPort{ + { + Name: lokiInternalHTTPPortName, + ContainerPort: internalHTTPPort, + Protocol: protocolTCP, + }, + }, + wantVolumeMounts: []corev1.VolumeMount{ + { + Name: serviceNameRulerHTTP(stackName), + ReadOnly: false, + MountPath: lokiServerHTTPTLSDir(), + }, + { + Name: serviceNameRulerGRPC(stackName), + ReadOnly: false, + MountPath: lokiServerGRPCTLSDir(), + }, + { + Name: signingCABundleName(stackName), + ReadOnly: false, + MountPath: caBundleDir, + }, + }, + wantVolumes: []corev1.Volume{ + { + Name: serviceNameRulerHTTP(stackName), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceNameRulerHTTP(stackName), + }, + }, + }, + { + Name: serviceNameRulerGRPC(stackName), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceNameRulerGRPC(stackName), + }, + }, + }, + { + Name: signingCABundleName(stackName), + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: signingCABundleName(stackName), + }, + }, + }, + }, + }, + }, + } + for _, test := range tt { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + objs, err := test.buildFunc(opts) + require.NoError(t, err) + + var pod *corev1.PodSpec + switch o := objs[0].(type) { + case *appsv1.Deployment: + pod = &o.Spec.Template.Spec + case *appsv1.StatefulSet: + pod = &o.Spec.Template.Spec + default: + t.Fatal("Wrong object type given") + } + + isEncryptionRelated := func(s string) bool { + return strings.Contains(s, "internal-server") || // Healthcheck server + strings.Contains(s, "client") || // Client certificates + strings.Contains(s, "-http") || // Serving HTTP certificates + strings.Contains(s, "-grpc") || // Serving GRPC certificates + strings.Contains(s, "ca") // Certificate authorities + } + + // Check args not missing + for _, arg := range test.wantArgs { + require.Contains(t, pod.Containers[0].Args, arg) + } + for _, arg := range pod.Containers[0].Args { + if isEncryptionRelated(arg) { + require.Contains(t, test.wantArgs, arg) + } + } + + // Check ports not missing + for _, port := range test.wantPorts { + require.Contains(t, pod.Containers[0].Ports, port) + } + + // Check mounts not missing + for _, mount := range test.wantVolumeMounts { + require.Contains(t, pod.Containers[0].VolumeMounts, mount) + } + for _, mount := range pod.Containers[0].VolumeMounts { + if isEncryptionRelated(mount.Name) { + require.Contains(t, test.wantVolumeMounts, mount) + } + } + + // Check volumes not missing + for _, volume := range test.wantVolumes { + require.Contains(t, pod.Volumes, volume) + } + for _, volume := range pod.Volumes { + if isEncryptionRelated(volume.Name) { + require.Contains(t, test.wantVolumes, volume) + } + } + }) + } +} diff --git a/operator/internal/manifests/var.go b/operator/internal/manifests/var.go index d211f98ebfc6..e6eff29f4b7b 100644 --- a/operator/internal/manifests/var.go +++ b/operator/internal/manifests/var.go @@ -13,14 +13,16 @@ import ( ) const ( - gossipPort = 7946 - httpPort = 3100 - grpcPort = 9095 - protocolTCP = "TCP" + gossipPort = 7946 + httpPort = 3100 + internalHTTPPort = 3101 + grpcPort = 9095 + protocolTCP = "TCP" - lokiHTTPPortName = "metrics" - lokiGRPCPortName = "grpclb" - lokiGossipPortName = "gossip-ring" + lokiHTTPPortName = "metrics" + lokiInternalHTTPPortName = "healthchecks" + lokiGRPCPortName = "grpclb" + lokiGossipPortName = "gossip-ring" lokiLivenessPath = "/loki/api/v1/status/buildinfo" lokiReadinessPath = "/ready" @@ -63,6 +65,11 @@ const ( // labelJobComponent is a ServiceMonitor.Spec.JobLabel. labelJobComponent string = "loki.grafana.com/component" + // AnnotationCertRotationRequiredAt stores the point in time the last cert rotation happened + AnnotationCertRotationRequiredAt string = "loki.grafana.com/certRotationRequiredAt" + // AnnotationLokiConfigHash stores the last SHA1 hash of the loki configuration + AnnotationLokiConfigHash string = "loki.grafana.com/config-hash" + // LabelCompactorComponent is the label value for the compactor component LabelCompactorComponent string = "compactor" // LabelDistributorComponent is the label value for the distributor component @@ -84,10 +91,6 @@ const ( httpTLSDir = "/var/run/tls/http" // grpcTLSDir is the path that is mounted from the secret for TLS grpcTLSDir = "/var/run/tls/grpc" - // tlsCertFile is the file of the X509 server certificate file - tlsCertFile = "tls.crt" - // tlsKeyFile is the file name of the server private key - tlsKeyFile = "tls.key" // LokiStackCABundleDir is the path that is mounted from the configmap for TLS caBundleDir = "/var/run/ca" // caFile is the file name of the certificate authority file @@ -102,9 +105,10 @@ var ( volumeFileSystemMode = corev1.PersistentVolumeFilesystem ) -func commonAnnotations(h string) map[string]string { +func commonAnnotations(configHash, rotationRequiredAt string) map[string]string { return map[string]string{ - "loki.grafana.com/config-hash": h, + AnnotationLokiConfigHash: configHash, + AnnotationCertRotationRequiredAt: rotationRequiredAt, } } @@ -193,6 +197,58 @@ func lokiConfigMapName(stackName string) string { return fmt.Sprintf("%s-config", stackName) } +func lokiServerGRPCTLSDir() string { + return path.Join(grpcTLSDir, "server") +} + +func lokiServerGRPCTLSCert() string { + return path.Join(lokiServerGRPCTLSDir(), corev1.TLSCertKey) +} + +func lokiServerGRPCTLSKey() string { + return path.Join(lokiServerGRPCTLSDir(), corev1.TLSPrivateKeyKey) +} + +func lokiServerHTTPTLSDir() string { + return path.Join(httpTLSDir, "server") +} + +func lokiServerHTTPTLSCert() string { + return path.Join(lokiServerHTTPTLSDir(), corev1.TLSCertKey) +} + +func lokiServerHTTPTLSKey() string { + return path.Join(lokiServerHTTPTLSDir(), corev1.TLSPrivateKeyKey) +} + +func gatewayServerHTTPTLSDir() string { + return path.Join(httpTLSDir, "server") +} + +func gatewayServerHTTPTLSCert() string { + return path.Join(gatewayServerHTTPTLSDir(), corev1.TLSCertKey) +} + +func gatewayServerHTTPTLSKey() string { + return path.Join(gatewayServerHTTPTLSDir(), corev1.TLSPrivateKeyKey) +} + +func gatewayUpstreamHTTPTLSDir() string { + return path.Join(httpTLSDir, "upstream") +} + +func gatewayUpstreamHTTPTLSCert() string { + return path.Join(gatewayUpstreamHTTPTLSDir(), corev1.TLSCertKey) +} + +func gatewayUpstreamHTTPTLSKey() string { + return path.Join(gatewayUpstreamHTTPTLSDir(), corev1.TLSPrivateKeyKey) +} + +func gatewayClientSecretName(stackName string) string { + return fmt.Sprintf("%s-gateway-client-http", stackName) +} + func serviceNameQuerierHTTP(stackName string) string { return fmt.Sprintf("%s-querier-http", stackName) } @@ -257,14 +313,46 @@ func serviceMonitorName(componentName string) string { return fmt.Sprintf("%s-monitor", componentName) } -func signingServiceSecretName(serviceName string) string { - return fmt.Sprintf("%s-tls", serviceName) -} - func signingCABundleName(stackName string) string { return fmt.Sprintf("%s-ca-bundle", stackName) } +func gatewaySigningCABundleName(gwName string) string { + return fmt.Sprintf("%s-ca-bundle", gwName) +} + +func gatewaySigningCADir() string { + return path.Join(caBundleDir, "server") +} + +func gatewaySigningCAPath() string { + return path.Join(gatewaySigningCADir(), caFile) +} + +func gatewayUpstreamCADir() string { + return path.Join(caBundleDir, "upstream") +} + +func gatewayUpstreamCAPath() string { + return path.Join(gatewayUpstreamCADir(), caFile) +} + +func gatewayTokenSecretName(gwName string) string { + return fmt.Sprintf("%s-token", gwName) +} + +func alertmanagerSigningCABundleName(rulerName string) string { + return fmt.Sprintf("%s-ca-bundle", rulerName) +} + +func alertmanagerUpstreamCADir() string { + return path.Join(caBundleDir, "alertmanager") +} + +func alertmanagerUpstreamCAPath() string { + return path.Join(alertmanagerUpstreamCADir(), caFile) +} + func signingCAPath() string { return path.Join(caBundleDir, caFile) } @@ -273,27 +361,82 @@ func fqdn(serviceName, namespace string) string { return fmt.Sprintf("%s.%s.svc.cluster.local", serviceName, namespace) } -// serviceMonitorTLSConfig returns the TLS configuration for service monitors. -func serviceMonitorTLSConfig(serviceName, namespace string) monitoringv1.TLSConfig { - return monitoringv1.TLSConfig{ - SafeTLSConfig: monitoringv1.SafeTLSConfig{ - // ServerName can be e.g. loki-distributor-http.openshift-logging.svc.cluster.local - ServerName: fqdn(serviceName, namespace), - }, - CAFile: PrometheusCAFile, +// lokiServiceMonitorEndpoint returns the lokistack endpoint for service monitors. +func lokiServiceMonitorEndpoint(stackName, portName, serviceName, namespace string, enableTLS bool) monitoringv1.Endpoint { + if enableTLS { + tlsConfig := monitoringv1.TLSConfig{ + SafeTLSConfig: monitoringv1.SafeTLSConfig{ + CA: monitoringv1.SecretOrConfigMap{ + ConfigMap: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: signingCABundleName(stackName), + }, + Key: caFile, + }, + }, + Cert: monitoringv1.SecretOrConfigMap{ + Secret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: serviceName, + }, + Key: corev1.TLSCertKey, + }, + }, + KeySecret: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: serviceName, + }, + Key: corev1.TLSPrivateKeyKey, + }, + // ServerName can be e.g. loki-distributor-http.openshift-logging.svc.cluster.local + ServerName: fqdn(serviceName, namespace), + }, + } + + return monitoringv1.Endpoint{ + Port: portName, + Path: "/metrics", + Scheme: "https", + TLSConfig: &tlsConfig, + } + } + + return monitoringv1.Endpoint{ + Port: portName, + Path: "/metrics", + Scheme: "http", } } -// serviceMonitorEndpoint returns the lokistack endpoint for service monitors. -func serviceMonitorEndpoint(portName, serviceName, namespace string, enableTLS bool) monitoringv1.Endpoint { +// gatewayServiceMonitorEndpoint returns the lokistack endpoint for service monitors. +func gatewayServiceMonitorEndpoint(gatewayName, portName, serviceName, namespace string, enableTLS bool) monitoringv1.Endpoint { if enableTLS { - tlsConfig := serviceMonitorTLSConfig(serviceName, namespace) + tlsConfig := monitoringv1.TLSConfig{ + SafeTLSConfig: monitoringv1.SafeTLSConfig{ + CA: monitoringv1.SecretOrConfigMap{ + ConfigMap: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: gatewaySigningCABundleName(gatewayName), + }, + Key: caFile, + }, + }, + // ServerName can be e.g. lokistack-dev-gateway-http.openshift-logging.svc.cluster.local + ServerName: fqdn(serviceName, namespace), + }, + } + return monitoringv1.Endpoint{ - Port: portName, - Path: "/metrics", - Scheme: "https", - BearerTokenFile: BearerTokenFile, - TLSConfig: &tlsConfig, + Port: portName, + Path: "/metrics", + Scheme: "https", + BearerTokenSecret: corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: gatewayTokenSecretName(gatewayName), + }, + Key: corev1.ServiceAccountTokenKey, + }, + TLSConfig: &tlsConfig, } } diff --git a/operator/main.go b/operator/main.go index 4c258abfc8ed..f7049495464e 100644 --- a/operator/main.go +++ b/operator/main.go @@ -102,25 +102,25 @@ func main() { if err = (&lokictrl.LokiStackReconciler{ Client: mgr.GetClient(), - Log: logger.WithName("controllers").WithName("LokiStack"), + Log: logger.WithName("controllers").WithName("lokistack"), Scheme: mgr.GetScheme(), FeatureGates: ctrlCfg.Gates, }).SetupWithManager(mgr); err != nil { - logger.Error(err, "unable to create controller", "controller", "LokiStack") + logger.Error(err, "unable to create controller", "controller", "lokistack") os.Exit(1) } if ctrlCfg.Gates.LokiStackWebhook { if err = (&lokiv1.LokiStack{}).SetupWebhookWithManager(mgr); err != nil { - logger.Error(err, "unable to create webhook", "webhook", "LokiStack") + logger.Error(err, "unable to create webhook", "webhook", "lokistack") os.Exit(1) } } if err = (&lokictrl.AlertingRuleReconciler{ Client: mgr.GetClient(), - Log: logger.WithName("controllers").WithName("AlertingRule"), + Log: logger.WithName("controllers").WithName("alertingrule"), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { - logger.Error(err, "unable to create controller", "controller", "AlertingRule") + logger.Error(err, "unable to create controller", "controller", "alertingrule") os.Exit(1) } if ctrlCfg.Gates.AlertingRuleWebhook { @@ -130,16 +130,16 @@ func main() { } if err = v.SetupWebhookWithManager(mgr); err != nil { - logger.Error(err, "unable to create webhook", "webhook", "AlertingRule") + logger.Error(err, "unable to create webhook", "webhook", "alertingrule") os.Exit(1) } } if err = (&lokictrl.RecordingRuleReconciler{ Client: mgr.GetClient(), - Log: logger.WithName("controllers").WithName("RecordingRule"), + Log: logger.WithName("controllers").WithName("recordingrule"), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { - logger.Error(err, "unable to create controller", "controller", "RecordingRule") + logger.Error(err, "unable to create controller", "controller", "recordingrule") os.Exit(1) } if ctrlCfg.Gates.RecordingRuleWebhook { @@ -149,7 +149,7 @@ func main() { } if err = v.SetupWebhookWithManager(mgr); err != nil { - logger.Error(err, "unable to create webhook", "webhook", "RecordingRule") + logger.Error(err, "unable to create webhook", "webhook", "recordingrule") os.Exit(1) } } @@ -157,9 +157,20 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { - logger.Error(err, "unable to create controller", "controller", "RulerConfig") + logger.Error(err, "unable to create controller", "controller", "rulerconfig") os.Exit(1) } + if ctrlCfg.Gates.BuiltInCertManagement.Enabled { + if err = (&lokictrl.CertRotationReconciler{ + Client: mgr.GetClient(), + Log: logger.WithName("controllers").WithName("certrotation"), + Scheme: mgr.GetScheme(), + FeatureGates: ctrlCfg.Gates, + }).SetupWithManager(mgr); err != nil { + logger.Error(err, "unable to create controller", "controller", "certrotation") + os.Exit(1) + } + } // +kubebuilder:scaffold:builder if err = mgr.AddHealthzCheck("health", healthz.Ping); err != nil { diff --git a/operator/quickstart.sh b/operator/quickstart.sh index 58bf08672955..70ccae1f7447 100755 --- a/operator/quickstart.sh +++ b/operator/quickstart.sh @@ -67,8 +67,9 @@ certificates() { kubectl -n cert-manager rollout status deployment cert-manager-webhook kubectl apply -f ./hack/addons_kind_certs.yaml - kubectl wait --timeout=180s --for=condition=ready certificate/lokistack-dev-ca-bundle - kubectl create configmap lokistack-dev-ca-bundle --from-literal service-ca.crt="$(kubectl get secret lokistack-dev-ca-bundle -o json | jq -r '.data."ca.crt"' | base64 -d -)" + kubectl wait --timeout=180s --for=condition=ready certificate/lokistack-dev-signing-ca + kubectl create configmap lokistack-dev-ca-bundle --from-literal service-ca.crt="$(kubectl get secret lokistack-dev-signing-ca -o json | jq -r '.data."ca.crt"' | base64 -d -)" + kubectl create configmap lokistack-dev-gateway-ca-bundle --from-literal service-ca.crt="$(kubectl get secret lokistack-dev-signing-ca -o json | jq -r '.data."ca.crt"' | base64 -d -)" } check() {