operator: Fix application tenant alertmanager configuration

operator/CHANGELOG.md

@@ -1,5 +1,6 @@
 ## Main
 
+- [9963](https://github.com/grafana/loki/pull/9963) **xperimental**: Fix application tenant alertmanager configuration
 - [9795](https://github.com/grafana/loki/pull/9795) **JoaoBraveCoding**: Add initContainer to zone aware components to gatekeep them from starting without the AZ annotation
 - [9503](https://github.com/grafana/loki/pull/9503) **shwetaap**: Add Pod annotations with node topology labels to support zone aware scheduling
 - [9930](https://github.com/grafana/loki/pull/9930) **periklis**: Use PodAntiAffinity for all components

operator/controllers/loki/lokistack_controller.go

@@ -280,7 +280,7 @@ func (r *LokiStackReconciler) enqueueForAlertManagerServices() handler.EventHand
 		var requests []reconcile.Request
 
 		if obj.GetName() == openshift.MonitoringSVCOperated &&
-			(obj.GetNamespace() == openshift.MonitoringUserwWrkloadNS ||
+			(obj.GetNamespace() == openshift.MonitoringUserWorkloadNS ||
 				obj.GetNamespace() == openshift.MonitoringNS) {
 
 			for _, stack := range lokiStacks.Items {

operator/internal/handlers/internal/openshift/alertmanager.go

@@ -36,7 +36,7 @@ func UserWorkloadAlertManagerSVCExists(ctx context.Context, stack lokiv1.LokiSta
 	}
 
 	var svc corev1.Service
-	key := client.ObjectKey{Name: openshift.MonitoringSVCOperated, Namespace: openshift.MonitoringUserwWrkloadNS}
+	key := client.ObjectKey{Name: openshift.MonitoringSVCOperated, Namespace: openshift.MonitoringUserWorkloadNS}
 
 	err := k.Get(ctx, key, &svc)
 	if err != nil && !apierrors.IsNotFound(err) {

operator/internal/manifests/config_test.go

@@ -1047,6 +1047,80 @@ func TestConfigOptions_RulerOverrides_OCPUserWorkloadOnlyEnabled(t *testing.T) {
 				},
 			},
 		},
+		{
+			desc: "openshift-logging mode with application override",
+			opts: Options{
+				Stack: lokiv1.LokiStackSpec{
+					Rules: &lokiv1.RulesSpec{
+						Enabled: true,
+					},
+					Limits: &lokiv1.LimitsSpec{
+						Tenants: map[string]lokiv1.LimitsTemplateSpec{
+							"application": {
+								QueryLimits: &lokiv1.QueryLimitSpec{
+									QueryTimeout: "5m",
+								},
+							},
+						},
+					},
+					Tenants: &lokiv1.TenantsSpec{
+						Mode: lokiv1.OpenshiftLogging,
+					},
+				},
+				Timeouts: testTimeoutConfig(),
+				Ruler: Ruler{
+					Spec: &lokiv1.RulerConfigSpec{
+						AlertManagerSpec: &lokiv1.AlertManagerSpec{
+							EnableV2: false,
+							DiscoverySpec: &lokiv1.AlertManagerDiscoverySpec{
+								EnableSRV:       false,
+								RefreshInterval: "2m",
+							},
+							Endpoints: []string{"http://my-alertmanager"},
+						},
+					},
+				},
+				OpenShiftOptions: openshift.Options{
+					BuildOpts: openshift.BuildOptions{
+						AlertManagerEnabled:             false,
+						UserWorkloadAlertManagerEnabled: true,
+					},
+				},
+			},
+			wantOptions: &config.AlertManagerConfig{
+				EnableV2:        false,
+				EnableDiscovery: false,
+				RefreshInterval: "2m",
+				Hosts:           "http://my-alertmanager",
+			},
+			wantOverridesOptions: map[string]config.LokiOverrides{
+				"application": {
+					Limits: lokiv1.LimitsTemplateSpec{
+						QueryLimits: &lokiv1.QueryLimitSpec{
+							QueryTimeout: "5m",
+						},
+					},
+					Ruler: config.RulerOverrides{
+						AlertManager: &config.AlertManagerConfig{
+							Hosts:           "https://_web._tcp.alertmanager-operated.openshift-user-workload-monitoring.svc",
+							EnableV2:        true,
+							EnableDiscovery: true,
+							RefreshInterval: "1m",
+							Notifier: &config.NotifierConfig{
+								TLS: config.TLSConfig{
+									ServerName: pointer.String("alertmanager-user-workload.openshift-user-workload-monitoring.svc.cluster.local"),
+									CAPath:     pointer.String("/var/run/ca/alertmanager/service-ca.crt"),
+								},
+								HeaderAuth: config.HeaderAuth{
+									Type:            pointer.String("Bearer"),
+									CredentialsFile: pointer.String("/var/run/secrets/kubernetes.io/serviceaccount/token"),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 		{
 			desc: "openshift-network mode",
 			opts: Options{

operator/internal/manifests/gateway_tenants.go

@@ -162,7 +162,7 @@ func ConfigureOptionsForMode(cfg *config.Options, opt Options) error {
 	case lokiv1.OpenshiftNetwork:
 		return openshift.ConfigureOptions(cfg, opt.OpenShiftOptions.BuildOpts.AlertManagerEnabled, false, "", "", "")
 	case lokiv1.OpenshiftLogging:
-		monitorServerName := fqdn(openshift.MonitoringSVCUserWorkload, openshift.MonitoringUserwWrkloadNS)
+		monitorServerName := fqdn(openshift.MonitoringSVCUserWorkload, openshift.MonitoringUserWorkloadNS)
 		return openshift.ConfigureOptions(
 			cfg,
 			opt.OpenShiftOptions.BuildOpts.AlertManagerEnabled,

operator/internal/manifests/openshift/configure.go

@@ -259,24 +259,18 @@ func configureDefaultMonitoringAM(configOpt *config.Options) error {
 }
 
 func configureUserWorkloadAM(configOpt *config.Options, token, caPath, monitorServerName string) error {
-	if len(configOpt.Overrides) == 0 {
+	if configOpt.Overrides == nil {
 		configOpt.Overrides = map[string]config.LokiOverrides{}
 	}
 
-	lokiOverrides, ok := configOpt.Overrides[tenantApplication]
-	if ok {
-		return nil
-	}
+	lokiOverrides := configOpt.Overrides[tenantApplication]
 
-	lokiOverrides = config.LokiOverrides{
-		Ruler: config.RulerOverrides{
-			AlertManager: &config.AlertManagerConfig{},
-		},
+	if lokiOverrides.Ruler.AlertManager != nil {
+		return nil
 	}
 
-	configOpt.Overrides[tenantApplication] = lokiOverrides
-	amOverride := &config.AlertManagerConfig{
-		Hosts:           fmt.Sprintf("https://_web._tcp.%s.%s.svc", MonitoringSVCOperated, MonitoringUserwWrkloadNS),
+	lokiOverrides.Ruler.AlertManager = &config.AlertManagerConfig{
+		Hosts:           fmt.Sprintf("https://_web._tcp.%s.%s.svc", MonitoringSVCOperated, MonitoringUserWorkloadNS),
 		EnableV2:        true,
 		EnableDiscovery: true,
 		RefreshInterval: "1m",
@@ -292,11 +286,6 @@ func configureUserWorkloadAM(configOpt *config.Options, token, caPath, monitorSe
 		},
 	}
 
-	if err := mergo.Merge(lokiOverrides.Ruler.AlertManager, amOverride); err != nil {
-		return kverrors.Wrap(err, "failed merging application tenant AlertManager config")
-	}
-
 	configOpt.Overrides[tenantApplication] = lokiOverrides
-
 	return nil
 }

operator/internal/manifests/openshift/var.go

@@ -45,7 +45,7 @@ var (
 	MonitoringSVCOperated = "alertmanager-operated"
 
 	MonitoringSVCUserWorkload = "alertmanager-user-workload"
-	MonitoringUserwWrkloadNS  = "openshift-user-workload-monitoring"
+	MonitoringUserWorkloadNS  = "openshift-user-workload-monitoring"
 )
 
 func authorizerRbacName(componentName string) string {