From a718769d11d170fb511c05a2d57b8c26e819faa7 Mon Sep 17 00:00:00 2001 From: Joao Morais Date: Mon, 7 Feb 2022 22:25:53 -0300 Subject: [PATCH] add starting implementation of gateway api v1alpha2 --- .../en/docs/configuration/gateway-api.md | 43 +- pkg/controller/cache.go | 41 +- pkg/converters/gateway/gateway.go | 423 +++++- pkg/converters/gateway/gateway_test.go | 1185 +++++++++++++++++ pkg/converters/helper_test/cachemock.go | 26 + pkg/converters/types/interfaces.go | 3 + 6 files changed, 1694 insertions(+), 27 deletions(-) create mode 100644 pkg/converters/gateway/gateway_test.go diff --git a/docs/content/en/docs/configuration/gateway-api.md b/docs/content/en/docs/configuration/gateway-api.md index 984849782..3d2741630 100644 --- a/docs/content/en/docs/configuration/gateway-api.md +++ b/docs/content/en/docs/configuration/gateway-api.md @@ -12,42 +12,44 @@ description: > The following steps configure the Kubernetes cluster and HAProxy Ingress to read and parse Gateway API resources: -* Manually install the Gateway API CRDs, see the Gateway API [documentation](https://gateway-api.sigs.k8s.io/v1alpha1/guides/getting-started/#installing-gateway-api-crds-manually) - * ... or simply `kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v0.3.0" | kubectl apply -f -` +* Manually install the Gateway API CRDs, see the Gateway API [documentation](https://gateway-api.sigs.k8s.io/v1alpha2/guides/getting-started/#installing-gateway-api-crds-manually) + * ... or simply `kubectl kustomize "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v0.4.1" | kubectl apply -f -` * Start (or restart) the controller See below the [getting started steps](#getting-started). ## Conformance -Gateway API v1alpha1 spec is partially implemented in the v0.13 release. The following list describes what is (or is not) supported: +Gateway API v1alpha2 spec is partially implemented in the v0.14 release. The following list describes what is (or is not) supported: * Target Services can be annotated with [Backend or Path scoped]({{% relref "keys#scope" %}}) configuration keys, this will continue to be supported. * Gateway API resources doesn't support annotations, this will continue to be unsupported, extensions to the Gateway API spec will be added in the extension points of the API. * Only the `GatewayClass`, `Gateway` and `HTTPRoute` resource definitions were implemented. * The controller doesn't implement partial parsing yet for Gateway API resources, changes should be a bit slow on clusters with thousands of Ingress, Gateway API resources or Services. * Gateway's Listener Port and Protocol wasn't implemented - Port uses the global [bind-port]({{% relref "keys#bind-port" %}}) configuration and Protocol is based on the presence or absence of the TLS attribute. -* Gateway's Route Namespace selector only supports `Same` or `All` namespaces. +* Gateway's Addresses wasn't implemented - binding addresses use the global [bind-ip-addr]({{% relref "keys#bind-ip-addr" %}}) configuration. * Gateway's Hostname only supports empty/absence of Hostname or a single `*`, any other string will override the HTTPRoute Hostnames configuration without any merging. * HTTPRoute's Matches doesn't support Headers. -* HTTPRoute's Rules and ForwardTo doesn't support Filters. +* HTTPRoute's Rules and BackendRefs don't support Filters. * Resources status aren't updated. -Version v1alpha2 should be partially implemented in v0.14 (beta version starting jan/22) and fully implemented in v0.15 (Q2'22). +Version v1alpha2 should be fully implemented in v0.15 (Q2'22). + +Version v1alpha1 support will be dropped in v0.16. ## Ingress -A single HAProxy Ingress deployment can manage Ingress and Gateway API resources in the same Kubernetes cluster. If the same hostname and path is declared in the Gateway API and Ingress, the Gateway API wins and a warning is logged. Ingress resources will continue to be supported in future controller versions, without side effects, and without the need to install the Gateway API CRDs. +A single HAProxy Ingress deployment can manage Ingress, and both v1alpha1 and v1alpha2 Gateway API resources in the same Kubernetes cluster. If the same hostname and path is declared in the Gateway API and Ingress, the Gateway API wins and a warning is logged. Ingress resources will continue to be supported in future controller versions, without side effects, and without the need to install the Gateway API CRDs. ## Getting started Add the following steps to the [Getting Started guide]({{% relref "/docs/getting-started" %}}) in order to expose the echoserver service along with the Gateway API: -[Manually install](https://gateway-api.sigs.k8s.io/guides/getting-started/#installing-gateway-api-crds-manually) the Gateway API CRDs: +[Manually install](https://gateway-api.sigs.k8s.io/v1alpha2/guides/getting-started/#installing-gateway-api-crds-manually) the Gateway API CRDs: ``` kubectl kustomize\ - "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v0.3.0" |\ + "github.com/kubernetes-sigs/gateway-api/config/crd?ref=v0.4.1" |\ kubectl apply -f - ``` @@ -61,12 +63,12 @@ kubectl --namespace default expose deployment echoserver --port=8080 A GatewayClass enables Gateways to be read and parsed by HAProxy Ingress. Create a GatewayClass with the following content: ```yaml -apiVersion: networking.x-k8s.io/v1alpha1 +apiVersion: gateway.networking.k8s.io/v1alpha2 kind: GatewayClass metadata: name: haproxy spec: - controller: haproxy-ingress.github.io/controller + controllerName: haproxy-ingress.github.io/controller ``` Gateways create listeners and allow to configure hostnames. Create a Gateway with the following content: @@ -74,7 +76,7 @@ Gateways create listeners and allow to configure hostnames. Create a Gateway wit Note: port and protocol attributes [have some limitations](#conformance). ```yaml -apiVersion: networking.x-k8s.io/v1alpha1 +apiVersion: gateway.networking.k8s.io/v1alpha2 kind: Gateway metadata: name: echoserver @@ -84,17 +86,16 @@ spec: listeners: - protocol: HTTP port: 80 - routes: - kind: HTTPRoute - selector: - matchLabels: - gateway: echo + name: echoserver-gw + allowedRoutes: + namespaces: + from: Same ``` HTTPRoutes configure the hostnames and target services. Create a HTTPRoute with the following content, changing `echoserver-from-gateway.local` to a hostname that resolves to a HAProxy Ingress node: ```yaml -apiVersion: networking.x-k8s.io/v1alpha1 +apiVersion: gateway.networking.k8s.io/v1alpha2 kind: HTTPRoute metadata: labels: @@ -102,11 +103,13 @@ metadata: name: echoserver namespace: default spec: + parentRefs: + - name: echoserver-gw hostnames: - echoserver-from-gateway.local rules: - - forwardTo: - - serviceName: echoserver + - backendRefs: + - name: echoserver port: 8080 ``` diff --git a/pkg/controller/cache.go b/pkg/controller/cache.go index 7d29336da..28462cafe 100644 --- a/pkg/controller/cache.go +++ b/pkg/controller/cache.go @@ -211,11 +211,12 @@ func (c *k8scache) hasGateway() bool { return c.listers.gatewayClassLister != nil } -var errGatewayDisabled = fmt.Errorf("Gateway API wasn't initialized") +var errGatewayA1Disabled = fmt.Errorf("Gateway API v1alpha1 wasn't initialized") +var errGatewayA2Disabled = fmt.Errorf("Gateway API v1alpha2 wasn't initialized") func (c *k8scache) GetGatewayA1(gatewayName string) (*gatewayv1alpha1.Gateway, error) { if !c.hasGatewayA1() { - return nil, errGatewayDisabled + return nil, errGatewayA1Disabled } namespace, name, err := cache.SplitMetaNamespaceKey(gatewayName) if err != nil { @@ -230,7 +231,7 @@ func (c *k8scache) GetGatewayA1(gatewayName string) (*gatewayv1alpha1.Gateway, e func (c *k8scache) GetGatewayA1List() ([]*gatewayv1alpha1.Gateway, error) { if !c.hasGatewayA1() { - return nil, errGatewayDisabled + return nil, errGatewayA1Disabled } gwList, err := c.listers.gatewayA1Lister.List(labels.Everything()) if err != nil { @@ -247,16 +248,33 @@ func (c *k8scache) GetGatewayA1List() ([]*gatewayv1alpha1.Gateway, error) { return validGwList[:i], nil } +func (c *k8scache) GetGatewayMap() (map[string]*gatewayv1alpha2.Gateway, error) { + if !c.hasGateway() { + return nil, errGatewayA2Disabled + } + gwList, err := c.listers.gatewayLister.List(labels.Everything()) + if err != nil { + return nil, err + } + validGwList := make(map[string]*gatewayv1alpha2.Gateway, len(gwList)) + for _, gw := range gwList { + if c.IsValidGateway(gw) { + validGwList[gw.Namespace+"/"+gw.Name] = gw + } + } + return validGwList, nil +} + func (c *k8scache) GetGatewayClassA1(className string) (*gatewayv1alpha1.GatewayClass, error) { if !c.hasGatewayA1() { - return nil, errGatewayDisabled + return nil, errGatewayA1Disabled } return c.listers.gatewayClassA1Lister.Get(className) } func (c *k8scache) GetGatewayClass(className string) (*gatewayv1alpha2.GatewayClass, error) { if !c.hasGateway() { - return nil, errGatewayDisabled + return nil, errGatewayA2Disabled } return c.listers.gatewayClassLister.Get(className) } @@ -271,7 +289,7 @@ func buildLabelSelector(match map[string]string) (labels.Selector, error) { func (c *k8scache) GetHTTPRouteA1List(namespace string, match map[string]string) ([]*gatewayv1alpha1.HTTPRoute, error) { if !c.hasGatewayA1() { - return nil, errGatewayDisabled + return nil, errGatewayA1Disabled } selector, err := buildLabelSelector(match) if err != nil { @@ -283,6 +301,13 @@ func (c *k8scache) GetHTTPRouteA1List(namespace string, match map[string]string) return c.listers.httpRouteA1Lister.List(selector) } +func (c *k8scache) GetHTTPRouteList() ([]*gatewayv1alpha2.HTTPRoute, error) { + if !c.hasGateway() { + return nil, errGatewayA2Disabled + } + return c.listers.httpRouteLister.List(labels.Everything()) +} + func (c *k8scache) GetService(defaultNamespace, serviceName string) (*api.Service, error) { namespace, name, err := c.buildResourceName(defaultNamespace, "service", serviceName, c.dynamicConfig.CrossNamespaceServices) if err != nil { @@ -310,6 +335,10 @@ func (c *k8scache) GetConfigMap(configMapName string) (*api.ConfigMap, error) { return c.listers.configMapLister.ConfigMaps(namespace).Get(name) } +func (c *k8scache) GetNamespace(name string) (*api.Namespace, error) { + return c.client.CoreV1().Namespaces().Get(c.ctx, name, metav1.GetOptions{}) +} + func (c *k8scache) GetEndpoints(service *api.Service) (*api.Endpoints, error) { return c.listers.endpointLister.Endpoints(service.Namespace).Get(service.Name) } diff --git a/pkg/converters/gateway/gateway.go b/pkg/converters/gateway/gateway.go index 783951f11..0ca64aeb1 100644 --- a/pkg/converters/gateway/gateway.go +++ b/pkg/converters/gateway/gateway.go @@ -17,8 +17,19 @@ limitations under the License. package gateway import ( + "fmt" + "sort" + "strconv" + + api "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" + convutils "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/utils" "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy" + hatypes "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy/types" "github.com/jcmoraisjr/haproxy-ingress/pkg/types" ) @@ -52,6 +63,9 @@ type converter struct { } func (c *converter) NeedFullSync() bool { + // Gateway API currently does not support partial parsing, so any change + // on resources tracked by any gateway API resource will return true to + // NeedFullSync(), which is the only way to Sync() start a reconciliation. links := c.tracker.QueryLinks(convtypes.TrackingLinks{ convtypes.ResourceSecret: c.changed.Links[convtypes.ResourceSecret], convtypes.ResourceService: c.changed.Links[convtypes.ResourceService], @@ -65,5 +79,412 @@ func (c *converter) Sync(full bool) { if !full { return } - // TODO implement + // TODO partial parsing + gateways, err := c.cache.GetGatewayMap() + if err != nil { + c.logger.Warn("error reading gateway list: %v", err) + return + } + httpRoutes, err := c.cache.GetHTTPRouteList() + if err != nil { + c.logger.Warn("error reading httpRoute list: %v", err) + return + } + sortHTTPRoutes(httpRoutes) + for _, httpRoute := range httpRoutes { + c.syncHTTPRoute(gateways, httpRoute) + } +} + +func sortHTTPRoutes(httpRoutes []*gatewayv1alpha2.HTTPRoute) { + sort.Slice(httpRoutes, func(i, j int) bool { + h1 := httpRoutes[i] + h2 := httpRoutes[j] + if h1.CreationTimestamp != h2.CreationTimestamp { + return h1.CreationTimestamp.Before(&h2.CreationTimestamp) + } + return h1.Namespace+"/"+h1.Name < h2.Namespace+"/"+h2.Name + }) +} + +// Source ... +// TODO reuse ingress' Source +type Source struct { + kind string + namespace string + name string +} + +func (s *Source) String() string { + return fmt.Sprintf("%s '%s/%s'", s.kind, s.namespace, s.name) +} + +var ( + gatewayGroup = gatewayv1alpha2.Group(gatewayv1alpha2.GroupName) + gatewayKind = gatewayv1alpha2.Kind("Gateway") + httpRouteKind = gatewayv1alpha2.Kind("HTTPRoute") +) + +func (c *converter) syncHTTPRoute(gateways map[string]*gatewayv1alpha2.Gateway, httpRoute *gatewayv1alpha2.HTTPRoute) { + httpRouteSource := &Source{ + kind: string(httpRouteKind), + namespace: httpRoute.Namespace, + name: httpRoute.Name, + } + for _, parentRef := range httpRoute.Spec.ParentRefs { + parentGroup := gatewayGroup + parentKind := gatewayKind + if parentRef.Group != nil && *parentRef.Group != "" { + parentGroup = *parentRef.Group + } + if parentRef.Kind != nil && *parentRef.Kind != "" { + parentKind = *parentRef.Kind + } + if parentGroup != gatewayGroup || parentKind != gatewayKind { + c.logger.Warn("ignoring unsupported Group/Kind reference on %s: %s/%s", + httpRouteSource, parentGroup, parentKind) + continue + } + namespace := httpRoute.Namespace + if parentRef.Namespace != nil && *parentRef.Namespace != "" { + namespace = string(*parentRef.Namespace) + } + gateway, found := gateways[namespace+"/"+string(parentRef.Name)] + if !found { + c.logger.Warn("%s references a gateway that was not found: %s/%s", + httpRouteSource, namespace, parentRef.Name) + continue + } + gatewaySource := &Source{ + kind: string(gatewayKind), + namespace: gateway.Namespace, + name: gateway.Name, + } + // TODO implement gateway.Spec.Addresses + err := c.syncHTTPRouteGateway(httpRouteSource, httpRoute, gatewaySource, gateway, parentRef.SectionName) + if err != nil { + c.logger.Warn("cannot attach %s to %s: %s", httpRouteSource, gatewaySource, err) + } + } +} + +func (c *converter) syncHTTPRouteGateway(httpRouteSource *Source, httpRoute *gatewayv1alpha2.HTTPRoute, gatewaySource *Source, gateway *gatewayv1alpha2.Gateway, sectionName *gatewayv1alpha2.SectionName) error { + for _, listener := range gateway.Spec.Listeners { + if sectionName != nil && *sectionName != listener.Name { + continue + } + if err := c.checkListenerAllowed(gatewaySource, httpRouteSource, &listener); err != nil { + c.logger.Warn("skipping attachment of %s to %s listener '%s': %s", + httpRouteSource, gatewaySource, listener.Name, err) + continue + } + for index, rule := range httpRoute.Spec.Rules { + // TODO implement rule.Filters + backend, services := c.createBackend(httpRouteSource, fmt.Sprintf("_rule%d", index), rule.BackendRefs) + if backend != nil { + passthrough := listener.TLS != nil && listener.TLS.Mode != nil && *listener.TLS.Mode == gatewayv1alpha2.TLSModePassthrough + if passthrough { + backend.ModeTCP = true + } + hostnames := c.filterHostnames(listener.Hostname, httpRoute.Spec.Hostnames) + hosts, pathLinks := c.createHTTPHosts(httpRouteSource, hostnames, rule.Matches, backend) + c.applyCertRef(gatewaySource, &listener, hosts) + if c.ann != nil { + c.ann.ReadAnnotations(backend, services, pathLinks) + } + } + } + } + return nil +} + +var errRouteNotAllowed = fmt.Errorf("listener does not allow the route") + +func (c *converter) checkListenerAllowed(gatewaySource, routeSource *Source, listener *gatewayv1alpha2.Listener) error { + if listener == nil || listener.AllowedRoutes == nil { + return errRouteNotAllowed + } + if err := checkListenerAllowedKind(routeSource, listener.AllowedRoutes.Kinds); err != nil { + return err + } + if err := c.checkListenerAllowedNamespace(gatewaySource, routeSource, listener.AllowedRoutes.Namespaces); err != nil { + return err + } + return nil +} + +func checkListenerAllowedKind(routeSource *Source, kinds []gatewayv1alpha2.RouteGroupKind) error { + if len(kinds) == 0 { + return nil + } + for _, kind := range kinds { + if (kind.Group == nil || *kind.Group == gatewayGroup) && kind.Kind == gatewayv1alpha2.Kind(routeSource.kind) { + return nil + } + } + return fmt.Errorf("listener does not allow route of Kind '%s'", routeSource.kind) +} + +func (c *converter) checkListenerAllowedNamespace(gatewaySource, routeSource *Source, namespaces *gatewayv1alpha2.RouteNamespaces) error { + if namespaces == nil || namespaces.From == nil { + return errRouteNotAllowed + } + if *namespaces.From == gatewayv1alpha2.NamespacesFromSame && routeSource.namespace == gatewaySource.namespace { + return nil + } + if *namespaces.From == gatewayv1alpha2.NamespacesFromAll { + return nil + } + if *namespaces.From == gatewayv1alpha2.NamespacesFromSelector { + if namespaces.Selector == nil { + return errRouteNotAllowed + } + selector, err := v1.LabelSelectorAsSelector(namespaces.Selector) + if err != nil { + return err + } + ns, err := c.cache.GetNamespace(routeSource.namespace) + if err != nil { + return err + } + if selector.Matches(labels.Set(ns.Labels)) { + return nil + } + } + return errRouteNotAllowed +} + +func (c *converter) createBackend(source *Source, index string, backendRefs []gatewayv1alpha2.HTTPBackendRef) (*hatypes.Backend, []*api.Service) { + if habackend := c.haproxy.Backends().FindBackend(source.namespace, source.name, index); habackend != nil { + return habackend, nil + } + type backend struct { + service gatewayv1alpha2.ObjectName + port string + epready []*convutils.Endpoint + cl convutils.WeightCluster + } + var backends []backend + var svclist []*api.Service + for _, back := range backendRefs { + if back.Port == nil { + // TODO implement nil back.Port + continue + } + // TODO implement back.Group + // TODO implement back.Kind + // TODO implement back.Namespace + svcName := source.namespace + "/" + string(back.Name) + c.tracker.TrackRefName([]convtypes.TrackingRef{ + {Context: convtypes.ResourceService, UniqueName: svcName}, + {Context: convtypes.ResourceEndpoints, UniqueName: svcName}, + }, convtypes.ResourceGateway, "gw") + svc, err := c.cache.GetService("", svcName) + if err != nil { + c.logger.Warn("skipping service '%s' on %s: %v", back.Name, source, err) + continue + } + svclist = append(svclist, svc) + portStr := strconv.Itoa(int(*back.Port)) + svcport := convutils.FindServicePort(svc, portStr) + if svcport == nil { + c.logger.Warn("skipping service '%s' on %s: port '%s' not found", back.Name, source, portStr) + continue + } + epready, _, err := convutils.CreateEndpoints(c.cache, svc, svcport) + if err != nil { + c.logger.Warn("skipping service '%s' on %s: %v", back.Name, source, err) + continue + } + weight := 1 + if back.Weight != nil { + weight = int(*back.Weight) + } + backends = append(backends, backend{ + service: back.Name, + port: svcport.TargetPort.String(), + epready: epready, + cl: convutils.WeightCluster{ + Weight: weight, + Length: len(epready), + }, + }) + // TODO implement back.BackendRef + // TODO implement back.Filters + } + if len(backends) == 0 { + return nil, nil + } + habackend := c.haproxy.Backends().AcquireBackend(source.namespace, source.name, index) + cl := make([]*convutils.WeightCluster, len(backends)) + for i := range backends { + cl[i] = &backends[i].cl + } + convutils.RebalanceWeight(cl, 128) + for i := range backends { + for _, addr := range backends[i].epready { + ep := habackend.AcquireEndpoint(addr.IP, addr.Port, addr.TargetRef) + ep.Weight = cl[i].Weight + } + } + return habackend, svclist +} + +func (c *converter) createHTTPHosts(source *Source, hostnames []gatewayv1alpha2.Hostname, matches []gatewayv1alpha2.HTTPRouteMatch, backend *hatypes.Backend) (hosts []*hatypes.Host, pathLinks []hatypes.PathLink) { + if backend.ModeTCP && len(matches) > 0 { + c.logger.Warn("ignoring match from %s: backend is TCP or SSL Passthrough", source) + matches = nil + } + if len(matches) == 0 { + matches = []gatewayv1alpha2.HTTPRouteMatch{{}} + } + for _, match := range matches { + var path string + var haMatch hatypes.MatchType + if match.Path != nil { + if match.Path.Value != nil { + path = *match.Path.Value + } + if match.Path.Type != nil { + switch *match.Path.Type { + case gatewayv1alpha2.PathMatchExact: + haMatch = hatypes.MatchExact + case gatewayv1alpha2.PathMatchPathPrefix: + haMatch = hatypes.MatchPrefix + case gatewayv1alpha2.PathMatchRegularExpression: + haMatch = hatypes.MatchRegex + } + } + } + if path == "" { + path = "/" + } + if haMatch == "" { + haMatch = hatypes.MatchPrefix + } + for _, hostname := range hostnames { + hstr := string(hostname) + if hstr == "" || hstr == "*" { + hstr = hatypes.DefaultHost + } + h := c.haproxy.Hosts().AcquireHost(hstr) + if h.FindPath(path, haMatch) != nil { + if backend.ModeTCP && h.SSLPassthrough() { + c.logger.Warn("skipping redeclared ssl-passthrough root path on %s", source) + continue + } + if !backend.ModeTCP && !h.SSLPassthrough() { + c.logger.Warn("skipping redeclared path '%s' type '%s' on %s", path, haMatch, source) + continue + } + } + h.TLS.UseDefaultCrt = false + h.AddPath(backend, path, haMatch) + c.handlePassthrough(path, h, backend, source) + hosts = append(hosts, h) + pathLinks = append(pathLinks, hatypes.CreatePathLink(hstr, path, haMatch)) + } + // TODO implement match.Headers + // TODO implement match.ExtensionRef + } + return hosts, pathLinks +} + +func (c *converter) handlePassthrough(path string, h *hatypes.Host, b *hatypes.Backend, source *Source) { + // Special handling for TLS passthrough due to current haproxy.Host limitation + // v0.15 will refactor haproxy.Host, allowing to remove this whole func + if path != "/" || (!b.ModeTCP && !h.SSLPassthrough()) { + // only matter if root path + // we also don't care if both present (b.ModeTCP) and past + // (h.SSLPassthrough()) passthrough isn't/wasn't configured + return + } + for _, hpath := range h.FindPath("/") { + modeTCP := hpath.Backend.ModeTCP + if modeTCP != nil && !*modeTCP { + // current path has a HTTP backend in the root path of a passthrough + // domain, and the current haproxy.Host implementation uses this as the + // target HTTPS backend. So we need to: + // + // 1. copy the backend ID to b.HTTPPassthroughBackend if not configured + if h.HTTPPassthroughBackend == "" { + if b.ModeTCP { + h.HTTPPassthroughBackend = hpath.Backend.ID + } else { + h.HTTPPassthroughBackend = b.ID + } + } else { + c.logger.Warn("skipping redeclared http root path on %s", source) + } + // and + // 2. remove it from the target HTTPS configuration + h.RemovePath(hpath) + } + } +} + +func (c *converter) filterHostnames(listenerHostname *gatewayv1alpha2.Hostname, routeHostnames []gatewayv1alpha2.Hostname) []gatewayv1alpha2.Hostname { + if listenerHostname == nil || *listenerHostname == "" || *listenerHostname == "*" { + if len(routeHostnames) == 0 { + return []gatewayv1alpha2.Hostname{"*"} + } + return routeHostnames + } + // TODO implement proper filter to wildcard based listenerHostnames -- `*.domain.local` + return []gatewayv1alpha2.Hostname{*listenerHostname} +} + +func (c *converter) applyCertRef(source *Source, listener *gatewayv1alpha2.Listener, hosts []*hatypes.Host) { + if listener.TLS == nil { + return + } + if listener.TLS.Mode != nil && *listener.TLS.Mode == gatewayv1alpha2.TLSModePassthrough { + for _, host := range hosts { + // backend was already changed to ModeTCP; hosts.match was already + // changed to root path only and a warning was already logged if needed + host.SetSSLPassthrough(true) + } + return + } + certRefs := listener.TLS.CertificateRefs + if len(certRefs) == 0 { + c.logger.Warn("skipping certificate reference on %s listener '%s': listener has no certificate reference", + source, listener.Name) + return + } + // TODO Support more certificates + if len(certRefs) > 1 { + err := fmt.Errorf("listener currently supports only the first referenced certificate") + c.logger.Warn("skipping one or more certificate references on %s listener '%s': %s", + source, listener.Name, err) + } + certRef := certRefs[0] + crtFile, err := c.readCertRef(source.namespace, certRef) + if err != nil { + c.logger.Warn("skipping certificate reference on %s listener '%s': %s", + source, listener.Name, err) + return + } + for _, host := range hosts { + if host.TLS.TLSHash != "" && host.TLS.TLSHash != crtFile.SHA1Hash { + c.logger.Warn("skipping certificate reference on %s listener '%s' for hostname '%s': a TLS certificate was already assigned", + source, listener.Name, host.Hostname) + continue + } + host.TLS.TLSCommonName = crtFile.CommonName + host.TLS.TLSFilename = crtFile.Filename + host.TLS.TLSHash = crtFile.SHA1Hash + } +} + +func (c *converter) readCertRef(namespace string, certRef *gatewayv1alpha2.SecretObjectReference) (crtFile convtypes.CrtFile, err error) { + if certRef.Group != nil && *certRef.Group != "" && *certRef.Group != "core" { + return crtFile, fmt.Errorf("unsupported Group '%s', supported groups are 'core' and ''", *certRef.Group) + } + if certRef.Kind != nil && *certRef.Kind != "" && *certRef.Kind != "Secret" { + return crtFile, fmt.Errorf("unsupported Kind '%s', the only supported kind is 'Secret'", *certRef.Kind) + } + // TODO implement certRef.Namespace + return c.cache.GetTLSSecretPath(namespace, string(certRef.Name), + []convtypes.TrackingRef{{Context: convtypes.ResourceGateway, UniqueName: "gw"}}) } diff --git a/pkg/converters/gateway/gateway_test.go b/pkg/converters/gateway/gateway_test.go new file mode 100644 index 000000000..3e1486de2 --- /dev/null +++ b/pkg/converters/gateway/gateway_test.go @@ -0,0 +1,1185 @@ +/* +Copyright 2022 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gateway + +import ( + "fmt" + "strings" + "testing" + + "github.com/kylelemons/godebug/diff" + api "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + gateway "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwapischeme "sigs.k8s.io/gateway-api/pkg/client/clientset/gateway/versioned/scheme" + + conv_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/helper_test" + "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/tracker" + convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" + "github.com/jcmoraisjr/haproxy-ingress/pkg/haproxy" + types_helper "github.com/jcmoraisjr/haproxy-ingress/pkg/types/helper_test" +) + +type testCaseSync struct { + id string + resConfig []string + config func(c *testConfig) + configTrack func(c *testConfig) + expFullSync bool + expDefaultHost string + expHosts string + expBackends string + expLogging string +} + +func TestSyncHTTPRouteCore(t *testing.T) { + otherGroup := gateway.Group("other.k8s.io") + runTestSync(t, []testCaseSync{ + { + id: "minimum", + config: func(c *testConfig) { + c.createGateway1("default/web", "l1") + c.createHTTPRoute1("default/web", "web", "echoserver:8080") + c.createService1("default/echoserver", "8080", "172.17.0.11") + }, + expDefaultHost: ` +hostname: +paths: +- path: / + match: prefix + backend: default_web__rule0 +`, + expBackends: ` +- id: default_web__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8080 + weight: 128 +`, + }, + { + id: "cross-namespace-1", + config: func(c *testConfig) { + c.createGateway1("ns1/web", "l1") + c.createHTTPRoute1("ns2/web", "ns1/web", "echoserver:8080") + c.createService1("ns2/echoserver", "8080", "172.17.0.11") + }, + expLogging: ` +WARN skipping attachment of HTTPRoute 'ns2/web' to Gateway 'ns1/web' listener 'l1': listener does not allow the route +`, + }, + { + id: "cross-namespace-2", + config: func(c *testConfig) { + g := c.createGateway1("ns1/web", "l1") + c.createHTTPRoute1("ns2/web", "ns1/web", "echoserver:8080") + c.createService1("ns2/echoserver", "8080", "172.17.0.11") + all := gateway.NamespacesFromAll + g.Spec.Listeners[0].AllowedRoutes.Namespaces.From = &all + }, + expDefaultHost: ` +hostname: +paths: +- path: / + match: prefix + backend: ns2_web__rule0 +`, + expBackends: ` +- id: ns2_web__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8080 + weight: 128 +`, + }, + { + id: "cross-namespace-3", + config: func(c *testConfig) { + c.createNamespace("ns2", "name=ns2") + c.createGateway1("ns1/web", "l1:name=ns1") + c.createHTTPRoute1("ns2/web", "ns1/web", "echoserver:8080") + c.createService1("ns2/echoserver", "8080", "172.17.0.11") + }, + expLogging: ` +WARN skipping attachment of HTTPRoute 'ns2/web' to Gateway 'ns1/web' listener 'l1': listener does not allow the route +`, + }, + { + id: "cross-namespace-4", + config: func(c *testConfig) { + c.createNamespace("ns2", "name=ns2") + c.createGateway1("ns1/web", "l1:name=ns2") + c.createHTTPRoute1("ns2/web", "ns1/web", "echoserver:8080") + c.createService1("ns2/echoserver", "8080", "172.17.0.11") + }, + expDefaultHost: ` +hostname: +paths: +- path: / + match: prefix + backend: ns2_web__rule0 +`, + expBackends: ` +- id: ns2_web__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8080 + weight: 128 +`, + }, + { + id: "cross-namespace-5", + config: func(c *testConfig) { + c.createGateway1("ns2/web", "l1") + c.createHTTPRoute1("ns2/web", "ns1/web", "echoserver:8080") + c.createService1("ns2/echoserver", "8080", "172.17.0.11") + }, + expLogging: ` +WARN HTTPRoute 'ns2/web' references a gateway that was not found: ns1/web +`, + }, + { + id: "allowed-kind-1", + config: func(c *testConfig) { + g := c.createGateway1("default/web", "l1,l2") + c.createHTTPRoute2("default/web1", "web:l1", "echoserver1:8080", "/app1") + c.createHTTPRoute2("default/web2", "web:l2", "echoserver2:8080", "/app2") + c.createService1("default/echoserver1", "8080", "172.17.0.11") + c.createService1("default/echoserver2", "8080", "172.17.0.12") + g.Spec.Listeners[0].AllowedRoutes.Kinds = []gateway.RouteGroupKind{ + {Kind: "HTTPRoute"}, + {Group: &otherGroup, Kind: "HTTPRoute"}, + } + g.Spec.Listeners[1].AllowedRoutes.Kinds = []gateway.RouteGroupKind{ + {Group: &gatewayGroup, Kind: "HTTPRoute"}, + {Group: &gatewayGroup, Kind: "OtherRoute"}, + } + }, + expDefaultHost: ` +hostname: +paths: +- path: /app2 + match: prefix + backend: default_web2__rule0 +- path: /app1 + match: prefix + backend: default_web1__rule0 +`, + expBackends: ` +- id: default_web1__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8080 + weight: 128 +- id: default_web2__rule0 + endpoints: + - ip: 172.17.0.12 + port: 8080 + weight: 128 +`, + }, + { + id: "allowed-kind-2", + config: func(c *testConfig) { + g := c.createGateway1("default/web", "l1") + c.createHTTPRoute1("default/web", "web:l1", "echoserver:8080") + c.createService1("default/echoserver", "8080", "172.17.0.11") + g.Spec.Listeners[0].AllowedRoutes.Kinds = []gateway.RouteGroupKind{ + {Kind: "OtherRoute"}, + {Group: &gatewayGroup, Kind: "OtherRoute"}, + {Group: &otherGroup, Kind: "HTTPRoute"}, + } + }, + expLogging: ` +WARN skipping attachment of HTTPRoute 'default/web' to Gateway 'default/web' listener 'l1': listener does not allow route of Kind 'HTTPRoute' +`, + }, + { + id: "multi-listener-1", + resConfig: []string{` +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: Gateway +metadata: + name: web + namespace: default +spec: + gatewayClassName: haproxy + listeners: + - name: h1 + allowedRoutes: + namespaces: + from: Same + - name: h2 + allowedRoutes: + namespaces: + from: Same +`}, + config: func(c *testConfig) { + c.createHTTPRoute2("default/web1", "web:h1", "echoserver1:8080", "/app1") + c.createHTTPRoute2("default/web2", "web:h2", "echoserver2:8080", "/app2") + c.createService1("default/echoserver1", "8080", "172.17.0.11") + c.createService1("default/echoserver2", "8080", "172.17.0.12") + }, + expDefaultHost: ` +hostname: +paths: +- path: /app2 + match: prefix + backend: default_web2__rule0 +- path: /app1 + match: prefix + backend: default_web1__rule0 +`, + expBackends: ` +- id: default_web1__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8080 + weight: 128 +- id: default_web2__rule0 + endpoints: + - ip: 172.17.0.12 + port: 8080 + weight: 128 +`, + }, + { + id: "tls-listener-no-refs-1", + config: func(c *testConfig) { + c.createGateway1("default/web", "l1,l2") + r1 := c.createHTTPRoute2("default/web1", "web:l1", "echoserver1:8080", "/") + r2 := c.createHTTPRoute2("default/web2", "web:l2", "echoserver2:8080", "/") + c.createService1("default/echoserver1", "8080", "172.17.0.11") + c.createService1("default/echoserver2", "8080", "172.17.0.12") + r1.Spec.Hostnames = []gateway.Hostname{"host1.local"} + r2.Spec.Hostnames = []gateway.Hostname{"host1.local"} + }, + expLogging: ` +WARN skipping redeclared path '/' type 'prefix' on HTTPRoute 'default/web2' +`, + expHosts: ` +- hostname: host1.local + paths: + - path: / + match: prefix + backend: default_web1__rule0 +`, + expBackends: ` +- id: default_web1__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8080 + weight: 128 +- id: default_web2__rule0 + endpoints: + - ip: 172.17.0.12 + port: 8080 + weight: 128 +`, + }, + }) +} + +func TestSyncHTTPRouteTracking(t *testing.T) { + runTestSync(t, []testCaseSync{ + { + id: "remove-secret-1", + config: func(c *testConfig) { + c.createGateway2("default/web", "l1", "crt") + c.createHTTPRoute1("default/web", "web", "echoserver:8080") + c.createService1("default/echoserver", "8080", "172.17.0.11") + c.cache.SecretTLSPath["default/crt"] = "/tls/crt.pem" + }, + configTrack: func(c *testConfig) { + c.cache.Changed.SecretsDel = append(c.cache.Changed.SecretsDel, c.createSecret1("default/crt1")) + }, + expFullSync: false, + }, + { + id: "remove-secret-2", + config: func(c *testConfig) { + c.createGateway2("default/web", "l1", "crt") + c.createHTTPRoute1("default/web", "web", "echoserver:8080") + c.createService1("default/echoserver", "8080", "172.17.0.11") + c.cache.SecretTLSPath["default/crt"] = "/tls/crt.pem" + }, + configTrack: func(c *testConfig) { + c.cache.Changed.SecretsDel = append(c.cache.Changed.SecretsDel, c.createSecret1("default/crt")) + }, + expFullSync: true, + }, + { + id: "add-secret-1", + config: func(c *testConfig) { + c.createGateway2("default/web", "l1", "crt") + c.createHTTPRoute1("default/web", "web", "echoserver:8080") + c.createService1("default/echoserver", "8080", "172.17.0.11") + }, + configTrack: func(c *testConfig) { + c.cache.Changed.SecretsAdd = append(c.cache.Changed.SecretsDel, c.createSecret1("default/crt1")) + }, + expFullSync: false, + expLogging: ` +WARN skipping certificate reference on Gateway 'default/web' listener 'l1': secret not found: 'default/crt' +`, + }, + { + id: "add-secret-2", + config: func(c *testConfig) { + c.createGateway2("default/web", "l1", "crt") + c.createHTTPRoute1("default/web", "web", "echoserver:8080") + c.createService1("default/echoserver", "8080", "172.17.0.11") + }, + configTrack: func(c *testConfig) { + c.cache.Changed.SecretsAdd = append(c.cache.Changed.SecretsDel, c.createSecret1("default/crt")) + }, + expFullSync: true, + expLogging: ` +WARN skipping certificate reference on Gateway 'default/web' listener 'l1': secret not found: 'default/crt' +`, + }, + { + id: "update-secret-1", + config: func(c *testConfig) { + c.createGateway2("default/web", "l1", "crt") + c.createHTTPRoute1("default/web", "web", "echoserver:8080") + c.createService1("default/echoserver", "8080", "172.17.0.11") + c.cache.SecretTLSPath["default/crt"] = "/tls/crt.pem" + }, + configTrack: func(c *testConfig) { + c.cache.Changed.SecretsUpd = append(c.cache.Changed.SecretsUpd, c.createSecret1("default/crt")) + }, + expFullSync: true, + }, + { + id: "remove-service-1", + config: func(c *testConfig) { + c.createGateway1("default/web", "l1") + c.createHTTPRoute1("default/web", "web", "echoserver:8080") + c.createService1("default/echoserver", "8080", "172.17.0.11") + }, + configTrack: func(c *testConfig) { + svc, _ := c.createService1("default/echoserver1", "8080", "172.17.0.11") + c.cache.Changed.ServicesDel = append(c.cache.Changed.ServicesDel, svc) + }, + expFullSync: false, + }, + { + id: "remove-service-2", + config: func(c *testConfig) { + c.createGateway1("default/web", "l1") + c.createHTTPRoute1("default/web", "web", "echoserver:8080") + c.createService1("default/echoserver", "8080", "172.17.0.11") + }, + configTrack: func(c *testConfig) { + svc, _ := c.createService1("default/echoserver", "8080", "172.17.0.11") + c.cache.Changed.ServicesDel = append(c.cache.Changed.ServicesDel, svc) + }, + expFullSync: true, + }, + { + id: "add-service-1", + config: func(c *testConfig) { + c.createGateway1("default/web", "l1") + c.createHTTPRoute1("default/web", "web", "echoserver:8080") + }, + configTrack: func(c *testConfig) { + svc, _ := c.createService1("default/echoserver", "8080", "172.17.0.11") + c.cache.Changed.ServicesDel = append(c.cache.Changed.ServicesDel, svc) + }, + expFullSync: true, + expLogging: ` +WARN skipping service 'echoserver' on HTTPRoute 'default/web': service not found: 'default/echoserver' +`, + }, + { + id: "change-endpoint-1", + config: func(c *testConfig) { + c.createGateway1("default/web", "l1") + c.createHTTPRoute1("default/web", "web", "echoserver:8080") + c.createService1("default/echoserver", "8080", "172.17.0.11") + }, + configTrack: func(c *testConfig) { + _, ep := c.createService1("default/echoserver", "8080", "172.17.0.12") + c.cache.Changed.EndpointsNew = append(c.cache.Changed.EndpointsNew, ep) + }, + expFullSync: true, + }, + }) +} + +func TestSyncHTTPRouteWeight(t *testing.T) { + runTestSync(t, []testCaseSync{ + { + id: "multi-backend-weight-1", + resConfig: []string{` +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: HTTPRoute +metadata: + name: web + namespace: default +spec: + parentRefs: + - name: web + rules: + - backendRefs: + - name: echoserver1 + port: 8080 + weight: 4 + - name: echoserver2 + port: 8080 + weight: 1 +`}, + config: func(c *testConfig) { + c.createGateway1("default/web", "l1") + c.createService1("default/echoserver1", "8080", "172.17.0.11") + c.createService1("default/echoserver2", "8080", "172.17.0.12") + }, + expDefaultHost: ` +hostname: +paths: +- path: / + match: prefix + backend: default_web__rule0 +`, + expBackends: ` +- id: default_web__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8080 + weight: 256 + - ip: 172.17.0.12 + port: 8080 + weight: 64 +`, + }, + { + id: "multi-backend-weight-2", + resConfig: []string{` +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: HTTPRoute +metadata: + labels: + gateway: web + name: web + namespace: default +spec: + parentRefs: + - name: web + rules: + - backendRefs: + - name: echoserver1 + port: 8080 + weight: 4 + - name: echoserver2 + port: 8080 + weight: 1 +`}, + config: func(c *testConfig) { + c.createGateway1("default/web", "l1") + c.createService1("default/echoserver1", "8080", "172.17.0.11") + c.createService1("default/echoserver2", "8080", "172.17.0.12,172.17.0.13,172.17.0.14,172.17.0.15") + }, + expDefaultHost: ` +hostname: +paths: +- path: / + match: prefix + backend: default_web__rule0 +`, + expBackends: ` +- id: default_web__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8080 + weight: 256 + - ip: 172.17.0.12 + port: 8080 + weight: 16 + - ip: 172.17.0.13 + port: 8080 + weight: 16 + - ip: 172.17.0.14 + port: 8080 + weight: 16 + - ip: 172.17.0.15 + port: 8080 + weight: 16 +`, + }, + }) +} + +func TestSyncGatewayTLS(t *testing.T) { + defaultBackend := ` +- id: default_web__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8080 + weight: 128 +` + defaultHTTPHost := ` +hostname: +paths: +- path: / + match: prefix + backend: default_web__rule0 +` + defaultHTTPSHost := ` +hostname: +paths: +- path: / + match: prefix + backend: default_web__rule0 +tls: + tlsfilename: /tls/default/crt.pem +` + runTestSync(t, []testCaseSync{ + { + id: "tls-listener-missing-secret-1", + config: func(c *testConfig) { + c.createGateway2("default/web", "l1", "crt") + c.createHTTPRoute1("default/web", "web:l1", "echoserver:8080") + c.createService1("default/echoserver", "8080", "172.17.0.11") + }, + expLogging: ` +WARN skipping certificate reference on Gateway 'default/web' listener 'l1': secret not found: 'default/crt' +`, + expDefaultHost: defaultHTTPHost, + expBackends: defaultBackend, + }, + { + id: "tls-listener-1", + config: func(c *testConfig) { + c.createSecret1("default/crt") + c.createGateway2("default/web", "l1", "crt") + c.createHTTPRoute1("default/web", "web:l1", "echoserver:8080") + c.createService1("default/echoserver", "8080", "172.17.0.11") + }, + expDefaultHost: defaultHTTPSHost, + expBackends: defaultBackend, + }, + { + id: "tls-listener-more-refs-1", + config: func(c *testConfig) { + c.createSecret1("default/crt") + c.createSecret1("default/crt2") + c.createGateway2("default/web", "l1", "crt,crt2") + c.createHTTPRoute1("default/web", "web:l1", "echoserver:8080") + c.createService1("default/echoserver", "8080", "172.17.0.11") + }, + expLogging: ` +WARN skipping one or more certificate references on Gateway 'default/web' listener 'l1': listener currently supports only the first referenced certificate +`, + expDefaultHost: defaultHTTPSHost, + expBackends: defaultBackend, + }, + { + id: "tls-listener-no-refs-1", + config: func(c *testConfig) { + c.createSecret1("default/crt1") + g := c.createGateway2("default/web", "l1", "crt1") + c.createHTTPRoute1("default/web", "web:l1", "echoserver:8080") + c.createService1("default/echoserver", "8080", "172.17.0.11") + g.Spec.Listeners[0].TLS.CertificateRefs = nil + }, + expLogging: ` +WARN skipping certificate reference on Gateway 'default/web' listener 'l1': listener has no certificate reference +`, + expDefaultHost: defaultHTTPHost, + expBackends: defaultBackend, + }, + { + id: "tls-listener-reassign-crt-1", + config: func(c *testConfig) { + c.createSecret1("default/crt1") + c.createSecret1("default/crt2") + c.createGateway2("default/web", "l1,l2", "crt1").Spec.Listeners[1].TLS.CertificateRefs[0].Name = "crt2" + r1 := c.createHTTPRoute1("default/web1", "web:l1", "echoserver1:8080") + r2 := c.createHTTPRoute1("default/web2", "web:l2", "echoserver2:8080") + c.createService1("default/echoserver1", "8080", "172.17.0.11") + c.createService1("default/echoserver2", "8080", "172.17.0.12") + r1.Spec.Hostnames = []gateway.Hostname{"host1.local"} + r2.Spec.Hostnames = []gateway.Hostname{"host1.local"} + }, + expLogging: ` +WARN skipping redeclared path '/' type 'prefix' on HTTPRoute 'default/web2' +`, + expHosts: ` +- hostname: host1.local + paths: + - path: / + match: prefix + backend: default_web1__rule0 + tls: + tlsfilename: /tls/default/crt1.pem +`, + expBackends: ` +- id: default_web1__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8080 + weight: 128 +- id: default_web2__rule0 + endpoints: + - ip: 172.17.0.12 + port: 8080 + weight: 128 +`, + }, + }) +} + +func TestSyncGatewayTLSPassthrough(t *testing.T) { + passthrough := gateway.TLSModePassthrough + runTestSync(t, []testCaseSync{ + { + id: "passthrough-1", + config: func(c *testConfig) { + g := c.createGateway1("default/web", "l1") + r := c.createHTTPRoute1("default/web", "web", "echoserver:8443") + c.createService1("default/echoserver", "8443", "172.17.0.11") + g.Spec.Listeners[0].TLS = &gateway.GatewayTLSConfig{Mode: &passthrough} + r.Spec.Hostnames = append(r.Spec.Hostnames, "domain.local") + }, + expHosts: ` +- hostname: domain.local + paths: + - path: / + match: prefix + backend: default_web__rule0 + passthrough: true +`, + expBackends: ` +- id: default_web__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8443 + weight: 128 + modetcp: true +`, + }, + { + id: "passthrough-and-http-first-1", + config: func(c *testConfig) { + g := c.createGateway1("default/web", "l1,l2") + r1 := c.createHTTPRoute2("default/web1", "web:l1", "echoserver1:8080", "/") + r2 := c.createHTTPRoute1("default/web2", "web:l2", "echoserver2:8443") + c.createService1("default/echoserver1", "8080", "172.17.0.11") + c.createService1("default/echoserver2", "8443", "172.17.0.12") + g.Spec.Listeners[1].TLS = &gateway.GatewayTLSConfig{Mode: &passthrough} + r1.Spec.Hostnames = []gateway.Hostname{"domain.local"} + r2.Spec.Hostnames = []gateway.Hostname{"domain.local"} + }, + expHosts: ` +- hostname: domain.local + paths: + - path: / + match: prefix + backend: default_web2__rule0 + passthrough: true + httppassback: default_web1__rule0 +`, + expBackends: ` +- id: default_web1__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8080 + weight: 128 +- id: default_web2__rule0 + endpoints: + - ip: 172.17.0.12 + port: 8443 + weight: 128 + modetcp: true +`, + }, + { + id: "passthrough-and-http-last-1", + config: func(c *testConfig) { + g := c.createGateway1("default/web", "l1,l2") + r1 := c.createHTTPRoute1("default/web1", "web:l1", "echoserver1:8443") + r2 := c.createHTTPRoute2("default/web2", "web:l2", "echoserver2:8080", "/") + c.createService1("default/echoserver1", "8443", "172.17.0.11") + c.createService1("default/echoserver2", "8080", "172.17.0.12") + g.Spec.Listeners[0].TLS = &gateway.GatewayTLSConfig{Mode: &passthrough} + r1.Spec.Hostnames = []gateway.Hostname{"domain.local"} + r2.Spec.Hostnames = []gateway.Hostname{"domain.local"} + }, + expHosts: ` +- hostname: domain.local + paths: + - path: / + match: prefix + backend: default_web1__rule0 + passthrough: true + httppassback: default_web2__rule0 +`, + expBackends: ` +- id: default_web1__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8443 + weight: 128 + modetcp: true +- id: default_web2__rule0 + endpoints: + - ip: 172.17.0.12 + port: 8080 + weight: 128 +`, + }, + { + id: "passthrough-and-http-dup-passthrough-1", + config: func(c *testConfig) { + g := c.createGateway1("default/web", "l1,l2,l3") + r1 := c.createHTTPRoute1("default/web1", "web:l1", "echoserver1:8443") + r2 := c.createHTTPRoute1("default/web2", "web:l2", "echoserver2:8443") + r3 := c.createHTTPRoute2("default/web3", "web:l3", "echoserver3:8080", "/") + c.createService1("default/echoserver1", "8443", "172.17.0.11") + c.createService1("default/echoserver2", "8443", "172.17.0.12") + c.createService1("default/echoserver3", "8080", "172.17.0.13") + g.Spec.Listeners[0].TLS = &gateway.GatewayTLSConfig{Mode: &passthrough} + g.Spec.Listeners[1].TLS = &gateway.GatewayTLSConfig{Mode: &passthrough} + r1.Spec.Hostnames = []gateway.Hostname{"domain.local"} + r2.Spec.Hostnames = []gateway.Hostname{"domain.local"} + r3.Spec.Hostnames = []gateway.Hostname{"domain.local"} + }, + expLogging: ` +WARN skipping redeclared ssl-passthrough root path on HTTPRoute 'default/web2' +`, + expHosts: ` +- hostname: domain.local + paths: + - path: / + match: prefix + backend: default_web1__rule0 + passthrough: true + httppassback: default_web3__rule0 +`, + expBackends: ` +- id: default_web1__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8443 + weight: 128 + modetcp: true +- id: default_web2__rule0 + endpoints: + - ip: 172.17.0.12 + port: 8443 + weight: 128 + modetcp: true +- id: default_web3__rule0 + endpoints: + - ip: 172.17.0.13 + port: 8080 + weight: 128 +`, + }, + { + id: "passthrough-and-http-dup-http-1", + config: func(c *testConfig) { + g := c.createGateway1("default/web", "l1,l2,l3") + r1 := c.createHTTPRoute1("default/web1", "web:l1", "echoserver1:8443") + r2 := c.createHTTPRoute2("default/web2", "web:l2", "echoserver2:8080", "/") + r3 := c.createHTTPRoute2("default/web3", "web:l3", "echoserver3:8080", "/") + c.createService1("default/echoserver1", "8443", "172.17.0.11") + c.createService1("default/echoserver2", "8080", "172.17.0.12") + c.createService1("default/echoserver3", "8080", "172.17.0.13") + g.Spec.Listeners[0].TLS = &gateway.GatewayTLSConfig{Mode: &passthrough} + r1.Spec.Hostnames = []gateway.Hostname{"domain.local"} + r2.Spec.Hostnames = []gateway.Hostname{"domain.local"} + r3.Spec.Hostnames = []gateway.Hostname{"domain.local"} + }, + expLogging: ` +WARN skipping redeclared http root path on HTTPRoute 'default/web3' +`, + expHosts: ` +- hostname: domain.local + paths: + - path: / + match: prefix + backend: default_web1__rule0 + passthrough: true + httppassback: default_web2__rule0 +`, + expBackends: ` +- id: default_web1__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8443 + weight: 128 + modetcp: true +- id: default_web2__rule0 + endpoints: + - ip: 172.17.0.12 + port: 8080 + weight: 128 +- id: default_web3__rule0 + endpoints: + - ip: 172.17.0.13 + port: 8080 + weight: 128 +`, + }, + { + id: "passthrough-and-http-path-1", + config: func(c *testConfig) { + g := c.createGateway1("default/web", "l1,l2") + r1 := c.createHTTPRoute2("default/web1", "web:l1", "echoserver1:8080", "/app") + r2 := c.createHTTPRoute1("default/web2", "web:l2", "echoserver2:8443") + c.createService1("default/echoserver1", "8080", "172.17.0.11") + c.createService1("default/echoserver2", "8443", "172.17.0.12") + g.Spec.Listeners[1].TLS = &gateway.GatewayTLSConfig{Mode: &passthrough} + r1.Spec.Hostnames = []gateway.Hostname{"domain.local"} + r2.Spec.Hostnames = []gateway.Hostname{"domain.local"} + }, + expHosts: ` +- hostname: domain.local + paths: + - path: /app + match: prefix + backend: default_web1__rule0 + - path: / + match: prefix + backend: default_web2__rule0 + passthrough: true +`, + expBackends: ` +- id: default_web1__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8080 + weight: 128 +- id: default_web2__rule0 + endpoints: + - ip: 172.17.0.12 + port: 8443 + weight: 128 + modetcp: true +`, + }, + { + id: "passthrough-with-match-1", + config: func(c *testConfig) { + g := c.createGateway1("default/web", "l1") + r1 := c.createHTTPRoute2("default/web", "web", "echoserver:8443", "/app") + c.createService1("default/echoserver", "8443", "172.17.0.11") + g.Spec.Listeners[0].TLS = &gateway.GatewayTLSConfig{Mode: &passthrough} + r1.Spec.Hostnames = []gateway.Hostname{"domain.local"} + }, + expLogging: ` +WARN ignoring match from HTTPRoute 'default/web': backend is TCP or SSL Passthrough +`, + expHosts: ` +- hostname: domain.local + paths: + - path: / + match: prefix + backend: default_web__rule0 + passthrough: true +`, + expBackends: ` +- id: default_web__rule0 + endpoints: + - ip: 172.17.0.11 + port: 8443 + weight: 128 + modetcp: true +`, + }, + }) +} + +func runTestSync(t *testing.T, testCases []testCaseSync) { + for _, test := range testCases { + c := setup(t) + + c.createGatewayResources(test.resConfig) + if test.config != nil { + test.config(c) + } + c.sync() + + if test.configTrack != nil { + c.hconfig.Commit() + test.configTrack(c) + conv := c.createConverter() + fullSync := conv.NeedFullSync() + if fullSync != test.expFullSync { + t.Errorf("%s: full sync differ, expected %t, actual: %t", test.id, test.expFullSync, fullSync) + } + } else { + if test.expDefaultHost == "" { + test.expDefaultHost = "[]" + } + if test.expHosts == "" { + test.expHosts = "[]" + } + if test.expBackends == "" { + test.expBackends = "[]" + } + c.compareConfigDefaultHost(test.id, test.expDefaultHost) + c.compareConfigHosts(test.id, test.expHosts) + c.compareConfigBacks(test.id, test.expBackends) + } + + c.logger.CompareLoggingID(test.id, test.expLogging) + + c.teardown() + } +} + +type testConfig struct { + t *testing.T + cache *conv_helper.CacheMock + logger *types_helper.LoggerMock + tracker convtypes.Tracker + hconfig haproxy.Config +} + +func setup(t *testing.T) *testConfig { + logger := types_helper.NewLoggerMock(t) + tracker := tracker.NewTracker() + c := &testConfig{ + t: t, + hconfig: haproxy.CreateInstance(logger, haproxy.InstanceOptions{}).Config(), + cache: conv_helper.NewCacheMock(tracker), + logger: logger, + tracker: tracker, + } + return c +} + +func (c *testConfig) teardown() { + c.logger.CompareLogging("") +} + +func (c *testConfig) sync() { + conv := c.createConverter() + conv.Sync(true) +} + +func (c *testConfig) createConverter() Config { + return NewGatewayConverter( + &convtypes.ConverterOptions{ + Cache: c.cache, + Logger: c.logger, + Tracker: c.tracker, + }, + c.hconfig, + c.cache.SwapChangedObjects(), + nil, + ) +} + +func (c *testConfig) createNamespace(name, labels string) *api.Namespace { + ns := &api.Namespace{} + ns.Name = name + ns.Labels = map[string]string{} + for _, label := range strings.Split(labels, ",") { + l := strings.Split(label, "=") + ns.Labels[l[0]] = l[1] + } + c.cache.NsList[name] = ns + return ns +} + +func (c *testConfig) createSecret1(secretName string) *api.Secret { + s := conv_helper.CreateSecret(secretName) + c.cache.SecretTLSPath[secretName] = "/tls/" + secretName + ".pem" + return s +} + +func (c *testConfig) createService1(name, port, ip string) (*api.Service, *api.Endpoints) { + svc, ep := conv_helper.CreateService(name, port, ip) + c.cache.SvcList = append(c.cache.SvcList, svc) + c.cache.EpList[name] = ep + return svc, ep +} + +func (c *testConfig) createGatewayClass1() *gateway.GatewayClass { + gc := CreateObject(` +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: GatewayClass +metadata: + name: haproxy +spec: + controller: haproxy-ingress.github.io/controller`).(*gateway.GatewayClass) + c.cache.GatewayClassList = append(c.cache.GatewayClassList, gc) + return gc +} + +func (c *testConfig) createGateway1(name, listeners string) *gateway.Gateway { + n := strings.Split(name, "/") + gw := CreateObject(` +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: Gateway +metadata: + name: ` + n[1] + ` + namespace: ` + n[0] + ` +spec: + gatewayClassName: haproxy + listeners: []`).(*gateway.Gateway) + for _, listener := range strings.Split(listeners, ",") { + l := gateway.Listener{} + var lname, lselector string + if i := strings.Index(listener, ":"); i >= 0 { + lname = listener[:i] + lselector = listener[i+1:] + } else { + lname = listener + } + l.Name = gateway.SectionName(lname) + from := gateway.NamespacesFromSame + var selector *v1.LabelSelector + if lselector != "" { + from = gateway.NamespacesFromSelector + selector = &v1.LabelSelector{ + MatchLabels: map[string]string{}, + } + for _, sel := range strings.Split(lselector, ";") { + s := strings.Split(sel, "=") + selector.MatchLabels[s[0]] = s[1] + } + } + l.AllowedRoutes = &gateway.AllowedRoutes{ + Namespaces: &gateway.RouteNamespaces{ + From: &from, + Selector: selector, + }, + } + gw.Spec.Listeners = append(gw.Spec.Listeners, l) + } + c.cache.GatewayList[name] = gw + return gw +} + +func (c *testConfig) createGateway2(name, listeners, secretName string) *gateway.Gateway { + gw := c.createGateway1(name, listeners) + for l := range gw.Spec.Listeners { + tls := &gateway.GatewayTLSConfig{} + for _, s := range strings.Split(secretName, ",") { + tls.CertificateRefs = append(tls.CertificateRefs, &gateway.SecretObjectReference{ + Name: gateway.ObjectName(s), + }) + } + gw.Spec.Listeners[l].TLS = tls + } + return gw +} + +func (c *testConfig) createHTTPRoute1(name, parent, service string) *gateway.HTTPRoute { + n := strings.Split(name, "/") + var pns, pn, ps string + if i := strings.Index(parent, "/"); i >= 0 { + pns = parent[:i] + pn = parent[i+1:] + } else { + pn = parent + } + if i := strings.Index(pn, ":"); i >= 0 { + ps = pn[i+1:] + pn = pn[:i] + } + svc := strings.Split(service, ":") + r := CreateObject(` +apiVersion: gateway.networking.k8s.io/v1alpha2 +kind: HTTPRoute +metadata: + name: ` + n[1] + ` + namespace: ` + n[0] + ` +spec: + parentRefs: + - name: ` + pn + ` + namespace: ` + pns + ` + sectionName: ` + ps + ` + rules: + - backendRefs: + - name: ` + svc[0] + ` + port: ` + svc[1]).(*gateway.HTTPRoute) + c.cache.HTTPRouteList = append(c.cache.HTTPRouteList, r) + return r +} + +func (c *testConfig) createHTTPRoute2(name, parent, service, paths string) *gateway.HTTPRoute { + r := c.createHTTPRoute1(name, parent, service) + prefix := gateway.PathMatchPathPrefix + for _, path := range strings.Split(paths, ",") { + p := path + match := gateway.HTTPRouteMatch{ + Path: &gateway.HTTPPathMatch{ + Type: &prefix, + Value: &p, + }, + } + r.Spec.Rules[0].Matches = append(r.Spec.Rules[0].Matches, match) + } + return r +} + +func (c *testConfig) createGatewayResources(res []string) { + for _, cfg := range res { + obj := CreateObject(cfg) + switch obj := obj.(type) { + case *gateway.Gateway: + c.cache.GatewayList[obj.Namespace+"/"+obj.Name] = obj + case *gateway.HTTPRoute: + c.cache.HTTPRouteList = append(c.cache.HTTPRouteList, obj) + case nil: + panic(fmt.Errorf("object is nil, cfg is %s", cfg)) + default: + panic(fmt.Errorf("unknown object type: %s", obj.GetObjectKind().GroupVersionKind().String())) + } + } +} + +func CreateObject(cfg string) runtime.Object { + decode := gwapischeme.Codecs.UniversalDeserializer().Decode + obj, _, err := decode([]byte(cfg), nil, nil) + if err != nil { + panic(err) + } + return obj +} + +func (c *testConfig) compareText(id string, actual, expected string) { + txt1 := "\n" + strings.Trim(expected, "\n") + txt2 := "\n" + strings.Trim(actual, "\n") + if txt1 != txt2 { + c.t.Errorf("diff on %s:%s", id, diff.Diff(txt1, txt2)) + } +} + +func (c *testConfig) compareConfigDefaultHost(id string, expected string) { + host := c.hconfig.Hosts().DefaultHost() + if host != nil { + c.compareText(id, conv_helper.MarshalHost(host), expected) + } else { + c.compareText(id, "[]", expected) + } +} + +func (c *testConfig) compareConfigHosts(id string, expected string) { + c.compareText(id, conv_helper.MarshalHosts(c.hconfig.Hosts().BuildSortedItems()...), expected) +} + +func (c *testConfig) compareConfigBacks(id string, expected string) { + c.compareText(id, conv_helper.MarshalBackendsWeight(c.hconfig.Backends().BuildSortedItems()...), expected) +} diff --git a/pkg/converters/helper_test/cachemock.go b/pkg/converters/helper_test/cachemock.go index d2c7f277a..1009a1eb0 100644 --- a/pkg/converters/helper_test/cachemock.go +++ b/pkg/converters/helper_test/cachemock.go @@ -26,6 +26,7 @@ import ( api "k8s.io/api/core/v1" networking "k8s.io/api/networking/v1" gatewayv1alpha1 "sigs.k8s.io/gateway-api/apis/v1alpha1" + gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" convtypes "github.com/jcmoraisjr/haproxy-ingress/pkg/converters/types" ) @@ -46,6 +47,11 @@ type CacheMock struct { GatewayA1ClassList []*gatewayv1alpha1.GatewayClass HTTPRouteA1List []*gatewayv1alpha1.HTTPRoute // + GatewayList map[string]*gatewayv1alpha2.Gateway + GatewayClassList []*gatewayv1alpha2.GatewayClass + HTTPRouteList []*gatewayv1alpha2.HTTPRoute + // + NsList map[string]*api.Namespace LookupList map[string][]net.IP EpList map[string]*api.Endpoints ConfigMapList map[string]*api.ConfigMap @@ -64,6 +70,8 @@ func NewCacheMock(tracker convtypes.Tracker) *CacheMock { tracker: tracker, Changed: &convtypes.ChangedObjects{}, SvcList: []*api.Service{}, + GatewayList: map[string]*gatewayv1alpha2.Gateway{}, + NsList: map[string]*api.Namespace{}, LookupList: map[string][]net.IP{}, EpList: map[string]*api.Endpoints{}, TermPodList: map[string][]*api.Pod{}, @@ -123,6 +131,11 @@ func (c *CacheMock) GetGatewayA1List() ([]*gatewayv1alpha1.Gateway, error) { return c.GatewayA1List, nil } +// GetGatewayMap ... +func (c *CacheMock) GetGatewayMap() (map[string]*gatewayv1alpha2.Gateway, error) { + return c.GatewayList, nil +} + // GetHTTPRouteA1List ... func (c *CacheMock) GetHTTPRouteA1List(namespace string, match map[string]string) ([]*gatewayv1alpha1.HTTPRoute, error) { routeMatch := func(route *gatewayv1alpha1.HTTPRoute) bool { @@ -145,6 +158,11 @@ func (c *CacheMock) GetHTTPRouteA1List(namespace string, match map[string]string return routes, nil } +// GetHTTPRouteList ... +func (c *CacheMock) GetHTTPRouteList() ([]*gatewayv1alpha2.HTTPRoute, error) { + return c.HTTPRouteList, nil +} + // GetService ... func (c *CacheMock) GetService(defaultNamespace, serviceName string) (*api.Service, error) { fullname := c.buildResourceName(defaultNamespace, serviceName) @@ -176,6 +194,14 @@ func (c *CacheMock) GetConfigMap(configMapName string) (*api.ConfigMap, error) { return nil, fmt.Errorf("configmap not found: %s", configMapName) } +// GetNamespace ... +func (c *CacheMock) GetNamespace(name string) (*api.Namespace, error) { + if ns, found := c.NsList[name]; found { + return ns, nil + } + return nil, fmt.Errorf("namespace not found: %s", name) +} + // GetTerminatingPods ... func (c *CacheMock) GetTerminatingPods(service *api.Service, track []convtypes.TrackingRef) ([]*api.Pod, error) { serviceName := service.Namespace + "/" + service.Name diff --git a/pkg/converters/types/interfaces.go b/pkg/converters/types/interfaces.go index 9247957bd..f1a3e0154 100644 --- a/pkg/converters/types/interfaces.go +++ b/pkg/converters/types/interfaces.go @@ -37,9 +37,12 @@ type Cache interface { GetGatewayA1(gatewayName string) (*gatewayv1alpha1.Gateway, error) GetGatewayA1List() ([]*gatewayv1alpha1.Gateway, error) GetHTTPRouteA1List(namespace string, match map[string]string) ([]*gatewayv1alpha1.HTTPRoute, error) + GetGatewayMap() (map[string]*gatewayv1alpha2.Gateway, error) + GetHTTPRouteList() ([]*gatewayv1alpha2.HTTPRoute, error) GetService(defaultNamespace, serviceName string) (*api.Service, error) GetEndpoints(service *api.Service) (*api.Endpoints, error) GetConfigMap(configMapName string) (*api.ConfigMap, error) + GetNamespace(name string) (*api.Namespace, error) GetTerminatingPods(service *api.Service, track []TrackingRef) ([]*api.Pod, error) GetPod(podName string) (*api.Pod, error) GetPodNamespace() string