ROX-27980: Support argocd reconciliation for routes (#2197)

Makefile

@@ -705,6 +705,8 @@ deploy/db:
 .PHONY: deploy/db
 
 # deploys the secrets required by the service to an OpenShift cluster
+# TLS cert and key are base64 encoded because make translates newlines in the shell command output into spaces which makes
+# pem certificates invalid
 deploy/secrets:
 	@oc process -f ./templates/secrets-template.yml --local \
 		-p DATABASE_HOST="fleet-manager-db" \
@@ -723,8 +725,8 @@ deploy/secrets:
 		-p SSO_CLIENT_ID="$(shell ([ -s './secrets/redhatsso-service.clientId' ] && [ -z '${SSO_CLIENT_ID}' ]) && cat ./secrets/redhatsso-service.clientId || echo '${SSO_CLIENT_ID}')" \
 		-p SSO_CLIENT_SECRET="$(shell ([ -s './secrets/redhatsso-service.clientSecret' ] && [ -z '${SSO_CLIENT_SECRET}' ]) && cat ./secrets/redhatsso-service.clientSecret || echo '${SSO_CLIENT_SECRET}')" \
 		-p CENTRAL_IDP_CLIENT_SECRET="$(shell ([ -s './secrets/central.idp-client-secret' ] && [ -z '${CENTRAL_IDP_CLIENT_SECRET}' ]) && cat ./secrets/central.idp-client-secret || echo '${CENTRAL_IDP_CLIENT_SECRET}')" \
-		-p CENTRAL_TLS_CERT="$(shell ([ -s './secrets/central-tls.crt' ] && [ -z '${CENTRAL_TLS_CERT}' ]) && cat ./secrets/central-tls.crt || echo '${CENTRAL_TLS_CERT}')" \
-		-p CENTRAL_TLS_KEY="$(shell ([ -s './secrets/central-tls.key' ] && [ -z '${CENTRAL_TLS_KEY}' ]) && cat ./secrets/central-tls.key || echo '${CENTRAL_TLS_KEY}')" \
+		-p CENTRAL_TLS_CERT="$(shell ([ -s './secrets/central-tls.crt' ] && [ -z '${CENTRAL_TLS_CERT}' ]) && (cat ./secrets/central-tls.crt || echo '${CENTRAL_TLS_CERT}') | base64)" \
+		-p CENTRAL_TLS_KEY="$(shell ([ -s './secrets/central-tls.key' ] && [ -z '${CENTRAL_TLS_KEY}' ]) && (cat ./secrets/central-tls.key || echo '${CENTRAL_TLS_KEY}') | base64)" \
 		| oc apply -f - -n $(NAMESPACE)
 .PHONY: deploy/secrets
 

fleetshard/pkg/central/reconciler/reconciler.go

@@ -79,6 +79,9 @@ const (
 	centralDbOverrideConfigMap = "central-db-override"
 	centralDeletePollInterval  = 5 * time.Second
 
+	centralCaTLSSecretName        = "managed-central-ca"        // pragma: allowlist secret
+	centralReencryptTLSSecretName = "managed-central-reencrypt" // pragma: allowlist secret
+
 	centralEncryptionKeySecretName = "central-encryption-key-chain" // pragma: allowlist secret
 
 	sensibleDeclarativeConfigSecretName = "cloud-service-sensible-declarative-configs" // pragma: allowlist secret
@@ -232,11 +235,21 @@ func (r *CentralReconciler) Reconcile(ctx context.Context, remoteCentral private
 		return nil, err
 	}
 
-	centralTLSSecretFound := true // pragma: allowlist secret
+	// central-tls secret provisioned by the ACS operator.
+	centralTLSSecretProvisioned := true // pragma: allowlist secret
 	if r.useRoutes {
-		if err := r.ensureRoutesExist(ctx, remoteCentral); err != nil {
+		if routesManagedByArgoCD(remoteCentral) {
+			if err := r.ensureRoutesDeleted(ctx, remoteCentral); err != nil {
+				return nil, err
+			}
+			err = r.reconcileIngressSecrets(ctx, remoteCentral)
+		} else {
+			err = r.ensureRoutesExist(ctx, remoteCentral)
+		}
+		if err != nil {
 			if k8s.IsCentralTLSNotFound(err) {
-				centralTLSSecretFound = false // pragma: allowlist secret
+				// Not considered as an error, waiting for the ACS operator to create it.
+				centralTLSSecretProvisioned = false // pragma: allowlist secret
 			} else {
 				return nil, errors.Wrap(err, "updating routes")
 			}
@@ -253,7 +266,7 @@ func (r *CentralReconciler) Reconcile(ctx context.Context, remoteCentral private
 		return nil, err
 	}
 
-	if !centralDeploymentReady || !centralTLSSecretFound {
+	if !centralDeploymentReady || !centralTLSSecretProvisioned {
 		if isRemoteCentralProvisioning(remoteCentral) && !needsReconcile { // no changes detected, wait until central become ready
 			return nil, ErrCentralNotChanged
 		}
@@ -283,6 +296,18 @@ func (r *CentralReconciler) Reconcile(ctx context.Context, remoteCentral private
 	return status, nil
 }
 
+func routesManagedByArgoCD(remoteCentral private.ManagedCentral) bool {
+	centralIngressEnabledValue, ok := remoteCentral.Spec.TenantResourcesValues["centralIngressEnabled"].(bool)
+	return ok && centralIngressEnabledValue
+}
+
+func (r *CentralReconciler) reconcileIngressSecrets(ctx context.Context, remoteCentral private.ManagedCentral) error {
+	if err := r.ensureCentralCASecretExists(ctx, remoteCentral.Metadata.Namespace); err != nil {
+		return err
+	}
+	return r.ensureReencryptSecretExists(ctx, remoteCentral)
+}
+
 func (r *CentralReconciler) restoreCentralSecrets(ctx context.Context, remoteCentral private.ManagedCentral) error {
 	restoreSecrets := []string{}
 	for _, secretName := range remoteCentral.Metadata.SecretsStored { // pragma: allowlist secret
@@ -519,15 +544,13 @@ func stringMapNeedsUpdating(desired, actual map[string]string) bool {
 }
 
 func (r *CentralReconciler) collectReconciliationStatus(ctx context.Context, remoteCentral *private.ManagedCentral) (*private.DataPlaneCentralStatus, error) {
-	remoteCentralNamespace := remoteCentral.Metadata.Namespace
-
 	status := readyStatus()
 	// Do not report routes statuses if:
 	// 1. Routes are not used on the cluster
 	// 2. Central request is in status "Ready" - assuming that routes are already reported and saved
 	if r.useRoutes && !isRemoteCentralReady(remoteCentral) {
 		var err error
-		status.Routes, err = r.getRoutesStatuses(ctx, remoteCentralNamespace)
+		status.Routes, err = r.getRoutesStatuses(ctx, remoteCentral)
 		if err != nil {
 			return nil, err
 		}
@@ -713,22 +736,33 @@ func isRemoteCentralReady(remoteCentral *private.ManagedCentral) bool {
 	return remoteCentral.RequestStatus == centralConstants.CentralRequestStatusReady.String()
 }
 
-func (r *CentralReconciler) getRoutesStatuses(ctx context.Context, namespace string) ([]private.DataPlaneCentralStatusRoutes, error) {
-	reencryptIngress, err := r.routeService.FindReencryptIngress(ctx, namespace)
+// getRoutesStatuses returns the list of the routes statuses required for Central to be ready.
+// Returns error if failed to find the routes ingresses or when AT LEAST ONE required route is unavailable.
+// This is because after Central is considered ready, fleet manager stores the route information.
+// Therefore, all required routes must be reported when Central is ready.
+func (r *CentralReconciler) getRoutesStatuses(ctx context.Context, central *private.ManagedCentral) ([]private.DataPlaneCentralStatusRoutes, error) {
+	unprocessedHosts := make(map[string]struct{}, 2)
+	unprocessedHosts[central.Spec.UiEndpoint.Host] = struct{}{}
+	unprocessedHosts[central.Spec.DataEndpoint.Host] = struct{}{}
+
+	ingresses, err := r.routeService.FindAdmittedIngresses(ctx, central.Metadata.Namespace)
 	if err != nil {
-		return nil, fmt.Errorf("obtaining ingress for reencrypt route: %w", err)
+		return nil, fmt.Errorf("obtaining ingresses for routes statuses: %w", err)
 	}
-	passthroughIngress, err := r.routeService.FindPassthroughIngress(ctx, namespace)
-	if err != nil {
-		return nil, fmt.Errorf("obtaining ingress for passthrough route: %w", err)
+	var routesStatuses []private.DataPlaneCentralStatusRoutes
+	for _, ingress := range ingresses {
+		if _, exists := unprocessedHosts[ingress.Host]; exists {
+			delete(unprocessedHosts, ingress.Host)
+			routesStatuses = append(routesStatuses, getRouteStatus(ingress))
+		}
+	}
+	if len(unprocessedHosts) != 0 {
+		return nil, fmt.Errorf("unable to find admitted ingress")
 	}
-	return []private.DataPlaneCentralStatusRoutes{
-		getRouteStatus(reencryptIngress),
-		getRouteStatus(passthroughIngress),
-	}, nil
+	return routesStatuses, nil
 }
 
-func getRouteStatus(ingress *openshiftRouteV1.RouteIngress) private.DataPlaneCentralStatusRoutes {
+func getRouteStatus(ingress openshiftRouteV1.RouteIngress) private.DataPlaneCentralStatusRoutes {
 	return private.DataPlaneCentralStatusRoutes{
 		Domain: ingress.Host,
 		Router: ingress.RouterCanonicalHostname,
@@ -997,6 +1031,49 @@ func (r *CentralReconciler) ensurePassthroughRouteExists(ctx context.Context, re
 	return nil
 }
 
+func (r *CentralReconciler) ensureRoutesDeleted(ctx context.Context, remoteCentral private.ManagedCentral) error {
+	namespace := remoteCentral.Metadata.Namespace
+	reencryptErr := r.routeService.DeleteReencryptRoute(ctx, namespace)
+	passthroughErr := r.routeService.DeletePassthroughRoute(ctx, namespace)
+
+	if reencryptErr != nil {
+		return fmt.Errorf("deleting reencrypt route for namespace %q: %w", namespace, reencryptErr)
+	}
+	if passthroughErr != nil {
+		return fmt.Errorf("deleting passthrough route for namespace %q: %w", namespace, passthroughErr)
+	}
+	return nil
+}
+
+func (r *CentralReconciler) ensureCentralCASecretExists(ctx context.Context, centralNamespace string) error {
+	centralTLSSecret, err := r.getSecret(centralNamespace, k8s.CentralTLSSecretName)
+	if err != nil {
+		if apiErrors.IsNotFound(err) {
+			return &k8s.SecretNotFound{SecretName: k8s.CentralTLSSecretName}
+		}
+		return err
+	}
+	return ensureSecretExists(ctx, r.client, centralNamespace, centralCaTLSSecretName, func(secret *corev1.Secret) error {
+		secret.Type = corev1.SecretTypeTLS
+		secret.Data = map[string][]byte{
+			corev1.TLSPrivateKeyKey: {},
+			corev1.TLSCertKey:       centralTLSSecret.Data["ca.pem"],
+		}
+		return nil
+	})
+}
+
+func (r *CentralReconciler) ensureReencryptSecretExists(ctx context.Context, central private.ManagedCentral) error {
+	return ensureSecretExists(ctx, r.client, central.Metadata.Namespace, centralReencryptTLSSecretName, func(secret *corev1.Secret) error {
+		secret.Type = corev1.SecretTypeTLS
+		secret.Data = map[string][]byte{
+			corev1.TLSPrivateKeyKey: []byte(central.Spec.UiEndpoint.Tls.Key),
+			corev1.TLSCertKey:       []byte(central.Spec.UiEndpoint.Tls.Cert),
+		}
+		return nil
+	})
+}
+
 func getTenantLabels(c private.ManagedCentral) map[string]string {
 	labels := map[string]string{
 		managedByLabelKey:    labelManagedByFleetshardValue,

fleetshard/pkg/central/reconciler/util.go

@@ -3,9 +3,10 @@ package reconciler
 import (
 	"context"
 	"fmt"
+	"strings"
+
 	"github.com/stackrox/acs-fleet-manager/fleetshard/pkg/k8s"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"strings"
 
 	centralClientPkg "github.com/stackrox/acs-fleet-manager/fleetshard/pkg/central/client"
 	"github.com/stackrox/rox/pkg/urlfmt"

fleetshard/pkg/k8s/route.go

@@ -6,9 +6,9 @@ import (
 
 	"github.com/stackrox/acs-fleet-manager/fleetshard/config"
 	"github.com/stackrox/rox/pkg/errox"
+	apiErrors "k8s.io/apimachinery/pkg/api/errors"
 
 	openshiftRouteV1 "github.com/openshift/api/route/v1"
-	"github.com/pkg/errors"
 	"github.com/stackrox/acs-fleet-manager/internal/dinosaur/pkg/api/private"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -78,28 +78,40 @@ func (s *RouteService) FindPassthroughRoute(ctx context.Context, namespace strin
 	return s.findRoute(ctx, namespace, centralPassthroughRouteName)
 }
 
-// FindReencryptIngress returns central reencrypt route ingress or error if not found.
+// FindReencryptIngress returns central reencrypt route ingress or nil if not found.
+// The error is returned when failed to get the route.
 func (s *RouteService) FindReencryptIngress(ctx context.Context, namespace string) (*openshiftRouteV1.RouteIngress, error) {
-	return s.findFirstAdmittedIngress(ctx, namespace, centralReencryptRouteName)
+	route, err := s.FindReencryptRoute(ctx, namespace)
+	if err != nil {
+		return nil, err
+	}
+	return findFirstAdmittedIngress(*route), nil
 }
 
-// FindPassthroughIngress returns central passthrough route ingress or error if not found.
-func (s *RouteService) FindPassthroughIngress(ctx context.Context, namespace string) (*openshiftRouteV1.RouteIngress, error) {
-	return s.findFirstAdmittedIngress(ctx, namespace, centralPassthroughRouteName)
+// FindAdmittedIngresses returns the list of admitted ingresses for a given namespace or error if the list could not be retrieved
+func (s *RouteService) FindAdmittedIngresses(ctx context.Context, namespace string) ([]openshiftRouteV1.RouteIngress, error) {
+	routes := &openshiftRouteV1.RouteList{}
+	if err := s.client.List(ctx, routes, ctrlClient.InNamespace(namespace)); err != nil {
+		return nil, fmt.Errorf("find admitted ingresses for namespace %s: %w", namespace, err)
+	}
+	var ingresses []openshiftRouteV1.RouteIngress
+	for _, route := range routes.Items {
+		admittedIngress := findFirstAdmittedIngress(route)
+		if admittedIngress != nil {
+			ingresses = append(ingresses, *admittedIngress)
+		}
+	}
+	return ingresses, nil
 }
 
-// findFirstAdmittedIngress returns first admitted ingress or error if not found
-func (s *RouteService) findFirstAdmittedIngress(ctx context.Context, namespace string, routeName string) (*openshiftRouteV1.RouteIngress, error) {
-	route, err := s.findRoute(ctx, namespace, routeName)
-	if err != nil {
-		return nil, fmt.Errorf("route not found")
-	}
+// findFirstAdmittedIngress returns first admitted ingress or nil if not found
+func findFirstAdmittedIngress(route openshiftRouteV1.Route) *openshiftRouteV1.RouteIngress {
 	for _, ingress := range route.Status.Ingress {
 		if isAdmitted(ingress) {
-			return &ingress, nil
+			return &ingress
 		}
 	}
-	return nil, fmt.Errorf("unable to find admitted ingress. route: %s/%s", route.GetNamespace(), route.GetName())
+	return nil
 }
 
 func isAdmitted(ingress openshiftRouteV1.RouteIngress) bool {
@@ -201,7 +213,7 @@ func (s *RouteService) UpdateReencryptRoute(ctx context.Context, route *openshif
 	}
 
 	if err := s.client.Update(ctx, updatedRoute); err != nil {
-		return errors.Wrapf(err, "updating reencrypt route")
+		return fmt.Errorf("updating reencrypt route: %w", err)
 	}
 
 	return nil
@@ -238,7 +250,7 @@ func (s *RouteService) UpdatePassthroughRoute(ctx context.Context, route *opensh
 	}
 
 	if err := s.client.Update(ctx, updatedRoute); err != nil {
-		return errors.Wrapf(err, "updating passthrough route")
+		return fmt.Errorf("updating passthrough route: %w", err)
 	}
 
 	return nil
@@ -287,6 +299,32 @@ func (s *RouteService) createCentralRoute(
 	return nil
 }
 
+// DeleteReencryptRoute deletes central reencrypt route for a given namespace
+func (s *RouteService) DeleteReencryptRoute(ctx context.Context, namespace string) error {
+	return s.deleteRoute(ctx, centralReencryptRouteName, namespace)
+}
+
+// DeletePassthroughRoute deletes central passthrough route for a given namespace
+func (s *RouteService) DeletePassthroughRoute(ctx context.Context, namespace string) error {
+	return s.deleteRoute(ctx, centralPassthroughRouteName, namespace)
+}
+
+func (s *RouteService) deleteRoute(ctx context.Context, name string, namespace string) error {
+	route := &openshiftRouteV1.Route{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      name,
+		},
+	}
+	if err := s.client.Delete(ctx, route); err != nil {
+		if apiErrors.IsNotFound(err) {
+			return nil // ok if not found
+		}
+		return fmt.Errorf("deleting route %s/%s: %w", namespace, name, err)
+	}
+	return nil
+}
+
 func (s *RouteService) findRoute(ctx context.Context, namespace string, routeName string) (*openshiftRouteV1.Route, error) {
 	route := &openshiftRouteV1.Route{
 		ObjectMeta: metav1.ObjectMeta{

fleetshard/pkg/testutils/k8s.go

@@ -164,14 +164,26 @@ func (t *ReconcileTracker) Create(gvr schema.GroupVersionResource, obj runtime.O
 		var multiErr *multierror.Error
 		multiErr = multierror.Append(multiErr, t.ObjectTracker.Create(centralsGVR, centralCR, destinationNamespace))
 		multiErr = multierror.Append(multiErr, t.ObjectTracker.Create(secretsGVR, newCentralTLSSecret(destinationNamespace), destinationNamespace))
-		multiErr = multierror.Append(multiErr, t.createRoute(t.newCentralRoute(destinationNamespace)))
-		multiErr = multierror.Append(multiErr, t.createRoute(t.newCentralMtlsRoute(destinationNamespace)))
 		multiErr = multierror.Append(multiErr, t.ObjectTracker.Create(deploymentGVR, NewCentralDeployment(destinationNamespace), destinationNamespace))
 		if err := multiErr.ErrorOrNil(); err != nil {
 			return fmt.Errorf("creating group version resource: %w", err)
 		}
 	}
 	return nil
+
+}
+
+// Update updates an existing object in the tracker in the specified namespace.
+func (t *ReconcileTracker) Update(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error {
+	if gvr == routesGVR {
+		route := obj.(*openshiftRouteV1.Route)
+		route.Status = t.admittedStatus(route.Name, route.Spec.Host)
+		return t.updateRoute(route)
+	}
+	if err := t.ObjectTracker.Update(gvr, obj, ns); err != nil {
+		return fmt.Errorf("adding GVR %q to reconcile tracker: %w", gvr, err)
+	}
+	return nil
 }
 
 func extractDestinationNamespaceFromArgoCDApp(app *argoCd.Application) string {
@@ -222,36 +234,16 @@ func (t *ReconcileTracker) createRoute(route *openshiftRouteV1.Route) error {
 	return errors.Wrapf(err, "create route")
 }
 
-func (t *ReconcileTracker) newCentralRoute(ns string) *openshiftRouteV1.Route {
-	host := fmt.Sprintf("central-%s.%s", ns, clusterDomain)
-	name := "central"
-	return &openshiftRouteV1.Route{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: ns,
-			Labels:    centralLabels,
-		},
-		Spec: openshiftRouteV1.RouteSpec{
-			Host: host,
-		},
-		Status: t.admittedStatus(name, host),
+func (t *ReconcileTracker) updateRoute(route *openshiftRouteV1.Route) error {
+	name := route.GetName()
+	if err := t.routeErrors[name]; err != nil {
+		return err
 	}
-}
-
-func (t *ReconcileTracker) newCentralMtlsRoute(ns string) *openshiftRouteV1.Route {
-	host := fmt.Sprintf("central.%s", ns)
-	name := "central-mtls"
-	return &openshiftRouteV1.Route{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      name,
-			Namespace: ns,
-			Labels:    centralLabels,
-		},
-		Spec: openshiftRouteV1.RouteSpec{
-			Host: host,
-		},
-		Status: t.admittedStatus(name, host),
+	if t.skipRoute[name] {
+		return nil
 	}
+	err := t.ObjectTracker.Update(routesGVR, route, route.GetNamespace())
+	return errors.Wrapf(err, "update route")
 }
 
 func (t *ReconcileTracker) admittedStatus(routeName string, host string) openshiftRouteV1.RouteStatus {