diff --git a/docs/content/en/docs/configuration/command-line.md b/docs/content/en/docs/configuration/command-line.md index 68c2b6ade..b0274a11b 100644 --- a/docs/content/en/docs/configuration/command-line.md +++ b/docs/content/en/docs/configuration/command-line.md @@ -306,6 +306,12 @@ Configure `--tcp-services-configmap` argument with `namespace/configmapname` res services and ports that HAProxy should listen to. Use the HAProxy's port number as the key of the ConfigMap. +{{% alert title="Note" %}} +Starting on v0.13, `--tcp-services-configmap` is deprecated. Use [`tcp-service-port`]({{% relref "keys#tcp-services" %}}) configuration key instead. + +The documentation refers to "ConfigMap based TCP" when taking about this configuration options, and it refers to "TCP Service" when talking about to the new, annotation based TCP configuration. +{{% /alert %}} + The value of the ConfigMap entry is a colon separated list of the following arguments: 1. `/`, mandatory, is the well known notation of the service that will receive incoming connections. @@ -343,6 +349,10 @@ HAProxy will listen 7 new ports: Note: Check interval was added in v0.10 and defaults to `2s`. All declared services has check interval enabled, except `3306` which disabled it. +See also: + +* [TCP Services]({{% relref "keys#tcp-services" %}}) configuration keys + --- ## --verify-hostname diff --git a/docs/content/en/docs/configuration/keys.md b/docs/content/en/docs/configuration/keys.md index 09eafb9e3..10625d853 100644 --- a/docs/content/en/docs/configuration/keys.md +++ b/docs/content/en/docs/configuration/keys.md @@ -94,7 +94,7 @@ metadata: Annotations are read in the following conditions: -* From classified `Ingress` resources, see about classification in the [Class matter](#class-matter) section. `Ingresses` accept keys from the `Host`, `Backend` and `Path` scopes. See about scopes [later](#scope) in this page. +* From classified `Ingress` resources, see about classification in the [Class matter](#class-matter) section. `Ingresses` accept keys from the `Host`, `Backend`, `Path` and `TCP` scopes. See about scopes [later](#scope) in this page. * From `Services` that classified Ingress resources are linking to. `Services` only accept keys from the `Backend` scope. A configuration key needs a prefix in front of its name to use as an annotation key. @@ -233,14 +233,14 @@ fast with HAProxy Ingress. # Scope -HAProxy Ingress configuration keys may be in one of four distinct scopes: -`Global`, `Host`, `Backend`, `Path`. A scope defines where a configuration key -can be declared and how it interacts with Ingress and Service resources. +HAProxy Ingress configuration keys may be in one of five distinct scopes: +`Global`, `Host`, `Backend`, `Path`, `TCP`. A scope defines where a configuration +key can be declared and how it interacts with Ingress and Service resources. Configuration keys declared in `Ingress` resources might conflict. More about -the scenarios in the `Host` and `Backend` scopes below. A warning will be logged -in the case of a conflict, and the used value will be of the Ingress resource -that was created first. +the scenarios in the `Host`, `Backend` and `TCP` scopes below. A warning will +be logged in the case of a conflict, and the used value will be of the Ingress +resource that was created first. ## Global @@ -274,6 +274,14 @@ Configuration keys of the Path scope can be declared in any ConfigMap as a defau value, or as Ingress or Service annotation. Configuration keys of the Path scope never conflict. +## TCP + +Defines configuration keys that bind on the port number of a TCP service. +Configuration keys of the TCP scope can be declared in any ConfigMap as a default +value, or as Ingress annotation. A conflict happens when the same TCP configuration +key with distinct values are declared in distinct Ingress resources but to the same +TCP port number. + # Keys The table below describes all supported configuration keys. @@ -329,7 +337,8 @@ The table below describes all supported configuration keys. | [`config-global`](#configuration-snippet) | multiline config for the global section | Global | | | [`config-proxy`](#configuration-snippet) | multiline config for any proxy | Global | | | [`config-sections`](#configuration-snippet) | multiline custom sections declaration | Global | | -| [`config-tcp`](#configuration-snippet) | multiline tcp-service config | Global | | +| [`config-tcp`](#configuration-snippet) | multiline ConfigMap based TCP config | Global | | +| [`config-tcp-service`](#configuration-snippet) | multiline TCP service config | TCP | | | [`cookie-key`](#affinity) | secret key | Global | `Ingress` | | [`cors-allow-credentials`](#cors) | [true\|false] | Path | | | [`cors-allow-headers`](#cors) | headers list | Path | | @@ -442,7 +451,10 @@ The table below describes all supported configuration keys. | [`syslog-format`](#syslog) | rfc5424\|rfc3164 | Global | `rfc5424` | | [`syslog-length`](#syslog) | maximum length | Global | `1024` | | [`syslog-tag`](#syslog) | syslog tag field string | Global | `ingress` | -| [`tcp-log-format`](#log-format) | tcp log format | Global | HAProxy default log format | +| [`tcp-log-format`](#log-format) | ConfigMap based TCP log format | Global | | +| [`tcp-service-log-format`](#log-format) | TCP service log format | TCP | HAProxy default log format | +| [`tcp-service-port`](#tcp-services) | TCP service port number | TCP | | +| [`tcp-service-proxy-protocol`](#proxy-protocol) | [true\|false] | TCP | `false` | | [`timeout-client`](#timeout) | time with suffix | Global | `50s` | | [`timeout-client-fin`](#timeout) | time with suffix | Global | `50s` | | [`timeout-connect`](#timeout) | time with suffix | Backend | `5s` | @@ -932,9 +944,9 @@ Define listening IPv4/IPv6 address on public HAProxy frontends. Since v0.10 the value changed from `*` to an empty string, which haproxy interprets in the same way and binds on all IPv4 address. -* `bind-ip-addr-tcp`: IP address of all TCP services declared on [`tcp-services`](#tcp-services-configmap) command-line option. +* `bind-ip-addr-tcp`: IP address of all ConfigMap based TCP services declared on [`tcp-services-configmap`]({{% relref "command-line#tcp-services-configmap" %}}) command-line option. * `bind-ip-addr-healthz`: IP address of the health check URL. -* `bind-ip-addr-http`: IP address of all HTTP/s frontends, port `:80` and `:443`, and also [`https-to-http-port`](#https-to-http-port) if declared. +* `bind-ip-addr-http`: IP address of all HTTP/s frontends, port `:80` and `:443`, and also [`fronting-proxy-port`](#fronting-proxy-port) if declared. * `bind-ip-addr-prometheus`: IP address of the haproxy's internal Prometheus exporter. * `bind-ip-addr-stats`: IP address of the statistics page. See also [`stats-port`](#stats). @@ -1075,15 +1087,16 @@ See also: ## Configuration snippet -| Configuration key | Scope | Default | Since | -|-------------------|-----------|----------|-------| -| `config-backend` | `Backend` | | | -| `config-defaults` | `Global` | | v0.8 | -| `config-frontend` | `Global` | | | -| `config-global` | `Global` | | | -| `config-proxy` | `Global` | | v0.13 | -| `config-sections` | `Global` | | v0.13 | -| `config-tcp` | `Global` | | v0.13 | +| Configuration key | Scope | Default | Since | +|----------------------|-----------|----------|-------| +| `config-backend` | `Backend` | | | +| `config-defaults` | `Global` | | v0.8 | +| `config-frontend` | `Global` | | | +| `config-global` | `Global` | | | +| `config-proxy` | `Global` | | v0.13 | +| `config-sections` | `Global` | | v0.13 | +| `config-tcp` | `Global` | | v0.13 | +| `config-tcp-service` | `TCP` | | v0.13 | Add HAProxy configuration snippet to the configuration file. Use multiline content to add more than one line of configuration. @@ -1094,7 +1107,8 @@ to add more than one line of configuration. * `config-global`: Adds a configuration snippet to the end of the HAProxy global section. * `config-proxy`: Adds a configuration snippet to any HAProxy proxy - listen, frontend or backend. It accepts a multi section configuration, where the name of the section is the name of a HAProxy proxy without the listen/frontend/backend prefix. A section whose proxy is not found is ignored. The content of each section should be indented, the first line without indentation is the start of a new section which will configure another proxy. * `config-sections`: Allows to declare new HAProxy sections. The configuration is used verbatim, without any indentation or validation. -* `config-tcp`: Adds a configuration snippet to the tcp-services sections. +* `config-tcp`: Adds a configuration snippet to the ConfigMap based TCP sections. +* `config-tcp-service`: Adds a configuration snippet to a TCP service section. Examples - ConfigMap: @@ -1140,7 +1154,7 @@ Examples - ConfigMap: capture request header X-User-Id len 32 ``` -Annotation: +Annotations: ```yaml annotations: @@ -1152,6 +1166,13 @@ Annotation: http-response cache-store icons if { var(txn.path) -m end .ico } ``` +```yaml + annotations: + ingress.kubernetes.io/config-tcp-service: | + timeout client 1m + timeout connect 15s +``` + --- ## Connection @@ -1590,12 +1611,13 @@ See also: ## Log format -| Configuration key | Scope | Default | Since | -|--------------------|----------|---------|-------| -| `auth-log-format` | `Global` | | v0.13 | -| `http-log-format` | `Global` | | | -| `https-log-format` | `Global` | | | -| `tcp-log-format` | `Global` | | | +| Configuration key | Scope | Default | Since | +|--------------------------|----------|---------|-------| +| `auth-log-format` | `Global` | | v0.13 | +| `http-log-format` | `Global` | | | +| `https-log-format` | `Global` | | | +| `tcp-log-format` | `Global` | | | +| `tcp-service-log-format` | `TCP` | | v0.13 | Customize the tcp, http or https log format using log format variables. Only used if [`syslog-endpoint`](#syslog) is also configured. @@ -1603,13 +1625,15 @@ Customize the tcp, http or https log format using log format variables. Only use * `auth-log-format`: log format of all auth external frontends. Use `default` to configure default HTTP log format, defaults to not log. * `http-log-format`: log format of all HTTP proxies, defaults to HAProxy default HTTP log format. * `https-log-format`: log format of TCP proxy used to inspect SNI extention. Use `default` to configure default TCP log format, defaults to not log. -* `tcp-log-format`: log format of TCP proxies, defaults to HAProxy default TCP log format. See also [TCP services configmap](#tcp-services-configmap) command-line option. +* `tcp-log-format`: log format of the ConfigMap based TCP proxies. Defaults to HAProxy default TCP log format. See also [`--tcp-services-configmap`]({{% relref "command-line#tcp-services-configmap" %}}) command-line option. +* `tcp-service-log-format`: log format of TCP frontends, configured via ingress resources and [`tcp-service-port`](#tcp-services) configuration key. Defaults to HAProxy default TCP log format. See also: * https://cbonte.github.io/haproxy-dconv/2.0/configuration.html#8.2.4 * [`syslog`](#syslog) * [Auth External](#auth-external) configuration keys. +* [TCP Services](#tcp-services) configuration keys. --- @@ -1825,15 +1849,17 @@ See also: ## Proxy protocol -| Configuration key | Scope | Default | Since | -|----------------------|-----------|---------|-------| -| `proxy-protocol` | `Backend` | `no` | | -| `use-proxy-protocol` | `Global` | `false` | | +| Configuration key | Scope | Default | Since | +|------------------------------|-----------|---------|-------| +| `proxy-protocol` | `Backend` | `no` | | +| `tcp-service-proxy-protocol` | `TCP` | `false` | v0.13 | +| `use-proxy-protocol` | `Global` | `false` | | Configures PROXY protocol in frontends and backends. * `proxy-protocol`: Define if the upstream backends support proxy protocol and what version of the protocol should be used. Supported values are `v1`, `v2`, `v2-ssl`, `v2-ssl-cn` or `no`. The default behavior if not declared is that the protocol is not supported by the backends and should not be used. -* `use-proxy-protocol`: Define if HAProxy is behind another proxy that use the PROXY protocol. If `true`, ports `80` and `443` will expect the PROXY protocol. The stats endpoint (defaults to port `1936`) has it's own [`stats-proxy-protocol`](#stats) configuration key. +* `use-proxy-protocol`: Define if HTTP services are behind another proxy that uses the PROXY protocol. If `true`, HTTP ports which defaults to `80` and `443` will expect the PROXY protocol, version 1 or 2. The stats endpoint (defaults to port `1936`) has its own [`stats-proxy-protocol`](#stats) configuration key. +* `tcp-service-proxy-protocol`: Define if the TCP service is behind another proxy that uses the PROXY protocol. Configures as `"true"` if the proxy should expect requests using the PROXY protocol, version 1 or 2. The default value is `"false"`. See also: @@ -2245,6 +2271,41 @@ See also: --- +## TCP Services + +| Configuration key | Scope | Default | Since | +|------------------------------|-------|---------|-------| +| `tcp-service-port` | `TCP` | | v0.13 | + +Configures a TCP proxy. + +* `tcp-service-port`: Defines the port number HAProxy should listen to. + +By default ingress resources configure HTTP services, and incoming requests are routed to backend servers based on hostnames and HTTP path. Whenever the `tcp-service-port` configuration key is added to an ingress resource, incoming requests are processed as TCP requests and the listening port number is used to route requests, using a dedicated frontend in tcp mode. Optionally, the TLS SNI extension can also be used to route incoming request if the hostname is declared in the ingress spec. + +Due to the limited data that can be inspected on TCP requests, a limited number of configuration keys work with TCP services: + +* `Backend` and `Path` scoped configuration keys work, provided that they are not HTTP related - eg [Cors](#cors) and [HSTS](#hsts) are ignored by TCP services, on the other hand [balance algorithm](#balance-algorithm), [Allow list](#allowlist) and [Blue/green](#blue-green) work just like in the HTTP requests counterpart. +* All `Global` configuration keys related with the whole haproxy process will also be applied to TCP services, like max connections or syslog configurations. +* All `Host` scoped configuration keys are currently unsupported + +Every TCP service port creates a dedicated haproxy frontend that can be [customized](#configuration-snippet) in three distinct ways: + +* `config-tcp-service` in the global ConfigMap, this will add the same configurations to all the TCP service frontends +* `config-tcp-service` as an Ingress annotation, this will add the snippet in one TCP service +* `config-proxy` in the global ConfigMap using `_front_tcp_` as the proxy name, see in the [configuration snippet](#configuration-snippet) documentation how it works + +{{% alert title="Note" %}} +The documentation continues to refer to the old, and now deprecated [`--tcp-services-configmap`]({{% relref "command-line#tcp-services-configmap" %}}) configuration options. Whenever we are talking about the deprecated option, we will refer it as the "ConfigMap based TCP". +{{% /alert %}} + +See also: + +* [`config-tcp-service`](#configuration-snippet) configuration key +* [`tcp-service-log-format`](#log-format) configuration key + +--- + ## Timeout | Configuration key | Scope | Default | Since | diff --git a/pkg/converters/ingress/annotations/host_test.go b/pkg/converters/ingress/annotations/host_test.go index 52577b25e..bef4b0f59 100644 --- a/pkg/converters/ingress/annotations/host_test.go +++ b/pkg/converters/ingress/annotations/host_test.go @@ -146,8 +146,10 @@ func TestTLSConfig(t *testing.T) { ingtypes.HostAuthTLSStrict: "true", }, expected: hatypes.HostTLSConfig{ - CAFilename: fakeCAFilename, - CAHash: fakeCAHash, + TLSConfig: hatypes.TLSConfig{ + CAFilename: fakeCAFilename, + CAHash: fakeCAHash, + }, }, logging: "ERROR error building TLS auth config on ingress 'system/ing1': secret not found: 'system/caerr'", }, @@ -157,8 +159,10 @@ func TestTLSConfig(t *testing.T) { ingtypes.HostAuthTLSSecret: "cafile", }, expected: hatypes.HostTLSConfig{ - CAFilename: "/path/ca.crt", - CAHash: "c0e1bf73caf75d7353cf3ecdd20ceb2f6fa1cab1", + TLSConfig: hatypes.TLSConfig{ + CAFilename: "/path/ca.crt", + CAHash: "c0e1bf73caf75d7353cf3ecdd20ceb2f6fa1cab1", + }, }, }, // 5 @@ -182,10 +186,11 @@ func TestTLSConfig(t *testing.T) { ingtypes.HostAuthTLSVerifyClient: "optional", }, expected: hatypes.HostTLSConfig{ - CAFilename: fakeCAFilename, - CAHash: fakeCAHash, - CAVerifyOptional: true, - }, + TLSConfig: hatypes.TLSConfig{ + CAFilename: fakeCAFilename, + CAHash: fakeCAHash, + CAVerifyOptional: true, + }}, logging: "ERROR error building TLS auth config on ingress 'system/ing1': secret not found: 'system/caerr'", }, // 8 @@ -196,10 +201,11 @@ func TestTLSConfig(t *testing.T) { ingtypes.HostAuthTLSVerifyClient: "optional", }, expected: hatypes.HostTLSConfig{ - CAFilename: "/path/ca.crt", - CAHash: "c0e1bf73caf75d7353cf3ecdd20ceb2f6fa1cab1", - CAVerifyOptional: true, - }, + TLSConfig: hatypes.TLSConfig{ + CAFilename: "/path/ca.crt", + CAHash: "c0e1bf73caf75d7353cf3ecdd20ceb2f6fa1cab1", + CAVerifyOptional: true, + }}, }, // 9 { @@ -208,10 +214,11 @@ func TestTLSConfig(t *testing.T) { ingtypes.HostAuthTLSVerifyClient: "optional", }, expected: hatypes.HostTLSConfig{ - CAFilename: "/path/ca.crt", - CAHash: "c0e1bf73caf75d7353cf3ecdd20ceb2f6fa1cab1", - CAVerifyOptional: true, - }, + TLSConfig: hatypes.TLSConfig{ + CAFilename: "/path/ca.crt", + CAHash: "c0e1bf73caf75d7353cf3ecdd20ceb2f6fa1cab1", + CAVerifyOptional: true, + }}, }, // 10 { @@ -229,8 +236,9 @@ func TestTLSConfig(t *testing.T) { ingtypes.HostSSLCiphers: "some-cipher-2:some-cipher-3", }, expected: hatypes.HostTLSConfig{ - Ciphers: "some-cipher-2:some-cipher-3", - }, + TLSConfig: hatypes.TLSConfig{ + Ciphers: "some-cipher-2:some-cipher-3", + }}, }, // 12 { @@ -248,8 +256,9 @@ func TestTLSConfig(t *testing.T) { ingtypes.HostSSLCipherSuites: "some-cipher-suite-2:some-cipher-suite-3", }, expected: hatypes.HostTLSConfig{ - CipherSuites: "some-cipher-suite-2:some-cipher-suite-3", - }, + TLSConfig: hatypes.TLSConfig{ + CipherSuites: "some-cipher-suite-2:some-cipher-suite-3", + }}, }, // 14 { @@ -267,8 +276,9 @@ func TestTLSConfig(t *testing.T) { ingtypes.HostTLSALPN: "h2", }, expected: hatypes.HostTLSConfig{ - ALPN: "h2", - }, + TLSConfig: hatypes.TLSConfig{ + ALPN: "h2", + }}, }, // 16 { @@ -276,8 +286,9 @@ func TestTLSConfig(t *testing.T) { ingtypes.HostSSLOptionsHost: "ssl-min-ver TLSv1.2", }, expected: hatypes.HostTLSConfig{ - Options: "ssl-min-ver TLSv1.2", - }, + TLSConfig: hatypes.TLSConfig{ + Options: "ssl-min-ver TLSv1.2", + }}, }, // 17 { @@ -288,8 +299,9 @@ func TestTLSConfig(t *testing.T) { ingtypes.HostSSLOptionsHost: "ssl-min-ver TLSv1.0 ssl-max-ver TLSv1.2", }, expected: hatypes.HostTLSConfig{ - Options: "ssl-min-ver TLSv1.0 ssl-max-ver TLSv1.2", - }, + TLSConfig: hatypes.TLSConfig{ + Options: "ssl-min-ver TLSv1.0 ssl-max-ver TLSv1.2", + }}, }, } source := &Source{Namespace: "system", Name: "ing1", Type: "ingress"} diff --git a/pkg/converters/ingress/annotations/updater.go b/pkg/converters/ingress/annotations/updater.go index 1851cc40a..2cd00112f 100644 --- a/pkg/converters/ingress/annotations/updater.go +++ b/pkg/converters/ingress/annotations/updater.go @@ -31,6 +31,8 @@ import ( // Updater ... type Updater interface { UpdateGlobalConfig(haproxyConfig haproxy.Config, mapper *Mapper) + UpdateTCPPortConfig(tcp *hatypes.TCPServicePort, mapper *Mapper) + UpdateTCPHostConfig(host *hatypes.TCPServiceHost, mapper *Mapper) UpdateHostConfig(host *hatypes.Host, mapper *Mapper) UpdateBackendConfig(backend *hatypes.Backend, mapper *Mapper) } @@ -163,6 +165,15 @@ func (c *updater) UpdateGlobalConfig(haproxyConfig haproxy.Config, mapper *Mappe c.buildGlobalTimeout(d) } +func (c *updater) UpdateTCPPortConfig(tcp *hatypes.TCPServicePort, mapper *Mapper) { + tcp.CustomConfig = utils.LineToSlice(mapper.Get(ingtypes.TCPConfigTCPService).Value) + tcp.LogFormat = mapper.Get(ingtypes.TCPTCPServiceLogFormat).Value + tcp.ProxyProt = mapper.Get(ingtypes.TCPTCPServiceProxyProto).Bool() +} + +func (c *updater) UpdateTCPHostConfig(host *hatypes.TCPServiceHost, mapper *Mapper) { +} + func (c *updater) UpdateHostConfig(host *hatypes.Host, mapper *Mapper) { data := &hostData{ host: host, diff --git a/pkg/converters/ingress/defaults.go b/pkg/converters/ingress/defaults.go index 892436c88..5edfe3242 100644 --- a/pkg/converters/ingress/defaults.go +++ b/pkg/converters/ingress/defaults.go @@ -30,6 +30,8 @@ const ( func createDefaults() map[string]string { return map[string]string{ + types.TCPTCPServiceLogFormat: "default", + // types.HostAuthTLSStrict: "false", types.HostServerRedirectCode: "302", types.HostSSLCiphers: defaultSSLCiphers, diff --git a/pkg/converters/ingress/ingress.go b/pkg/converters/ingress/ingress.go index d9e4dd2a8..563775387 100644 --- a/pkg/converters/ingress/ingress.go +++ b/pkg/converters/ingress/ingress.go @@ -70,6 +70,7 @@ func NewIngressConverter(options *ingtypes.ConverterOptions, haproxy haproxy.Con mapBuilder: annotations.NewMapBuilder(options.Logger, options.AnnotationPrefix+"/", defaultConfig), updater: annotations.NewUpdater(haproxy, options), globalConfig: annotations.NewMapBuilder(options.Logger, "", defaultConfig).NewMapper(), + tcpsvcAnnotations: map[*hatypes.TCPServicePort]*annotations.Mapper{}, hostAnnotations: map[*hatypes.Host]*annotations.Mapper{}, backendAnnotations: map[*hatypes.Backend]*annotations.Mapper{}, ingressClasses: map[string]*ingressClassConfig{}, @@ -89,6 +90,7 @@ type converter struct { mapBuilder *annotations.MapBuilder updater annotations.Updater globalConfig *annotations.Mapper + tcpsvcAnnotations map[*hatypes.TCPServicePort]*annotations.Mapper hostAnnotations map[*hatypes.Host]*annotations.Mapper backendAnnotations map[*hatypes.Backend]*annotations.Mapper ingressClasses map[string]*ingressClassConfig @@ -285,6 +287,16 @@ func (c *converter) syncPartial() { c.tracker.DeleteBackends(dirtyBacks) c.tracker.DeleteUserlists(dirtyUsers) c.tracker.DeleteStorages(dirtyStorages) + + // TCP services are currently in the host list due to how tracking is + // currently implemented. This is not a good solution because of their scopes - + // hosts and TCP services are managed by distinct entities in the haproxy model + // + // TODO Create a new tracker for services or another way to clean/remove + // backends during service updates. See also normalizeHostname() + var dirtyTCPServices []string + dirtyHosts, dirtyTCPServices = splitHostsAndTCPServices(dirtyHosts) + c.haproxy.TCPServices().RemoveAll(dirtyTCPServices) c.haproxy.Hosts().RemoveAll(dirtyHosts) c.haproxy.Frontend().RemoveAuthBackendByTarget(dirtyBacks) c.haproxy.Backends().RemoveAll(dirtyBacks) @@ -352,8 +364,9 @@ func (c *converter) trackAddedIngress() { c.tracker.TrackBackend(convtypes.IngressType, name, backend.BackendID()) } } + port, _ := strconv.Atoi(ing.Annotations[c.options.AnnotationPrefix+"/"+ingtypes.TCPTCPServicePort]) for _, rule := range ing.Spec.Rules { - c.tracker.TrackHostname(convtypes.IngressType, name, rule.Host) + c.tracker.TrackHostname(convtypes.IngressType, name, normalizeHostname(rule.Host, port)) if rule.HTTP != nil { for _, path := range rule.HTTP.Paths { backend := c.findBackend(ing.Namespace, &path.Backend) @@ -383,6 +396,39 @@ func (c *converter) findBackend(namespace string, backend *networking.IngressBac return c.haproxy.Backends().FindBackend(namespace, svcName, port.TargetPort.String()) } +// normalizeHostname adjusts the hostname according to the following rules: +// +// * empty hostnames are changed to `hatypes.DefaultHost` which has a +// special meaning in the hosts entity +// * hostnames for tcp services receive the port number to distinguish +// two tcp services without hostname. hostnames are preserved, making it +// a bit easier to introduce sni based routing. +// +// Hostnames are used as the tracking ID by backends and secrets. This design +// must be revisited - either evolving the tracking system, or abstracting +// how backends and secrets are tracked, or removing the tracking at all. +func normalizeHostname(hostname string, port int) string { + if hostname == "" { + hostname = hatypes.DefaultHost + } + if port > 0 { + return hostname + ":" + strconv.Itoa(port) + } + return hostname +} + +func splitHostsAndTCPServices(hostnames []string) (hosts, tcpServices []string) { + hosts = make([]string, 0, len(hostnames)) + for _, h := range hostnames { + if strings.Index(h, ":") >= 0 { + tcpServices = append(tcpServices, h) + } else { + hosts = append(hosts, h) + } + } + return hosts, tcpServices +} + func sortIngress(ingress []*networking.Ingress) { sort.Slice(ingress, func(i, j int) bool { i1 := ingress[i] @@ -395,30 +441,35 @@ func sortIngress(ingress []*networking.Ingress) { } func (c *converter) syncIngress(ing *networking.Ingress) { - fullIngName := fmt.Sprintf("%s/%s", ing.Namespace, ing.Name) + annTCP, annHost, annBack := c.readAnnotations(ing.Annotations) + tcpServicePort, _ := strconv.Atoi(annTCP[ingtypes.TCPTCPServicePort]) + if tcpServicePort == 0 { + c.syncIngressHTTP(ing, annHost, annBack) + } else { + c.syncIngressTCP(ing, tcpServicePort, annTCP, annBack) + } +} + +func (c *converter) syncIngressHTTP(ing *networking.Ingress, annHost, annBack map[string]string) { source := &annotations.Source{ Namespace: ing.Namespace, Name: ing.Name, Type: "ingress", } - annHost, annBack := c.readAnnotations(ing.Annotations) if ing.Spec.DefaultBackend != nil { svcName, svcPort, err := readServiceNamePort(ing.Spec.DefaultBackend) if err == nil { err = c.addDefaultHostBackend(source, ing.Namespace+"/"+svcName, svcPort, annHost, annBack) } if err != nil { - c.logger.Warn("skipping default backend of ingress '%s': %v", fullIngName, err) + c.logger.Warn("skipping default backend of %v: %v", source, err) } } for _, rule := range ing.Spec.Rules { if rule.HTTP == nil { continue } - hostname := rule.Host - if hostname == "" { - hostname = hatypes.DefaultHost - } + hostname := normalizeHostname(rule.Host, 0) ingressClass := c.readIngressClass(source, hostname, ing.Spec.IngressClassName) host := c.addHost(hostname, source, annHost) for _, path := range rule.HTTP.Paths { @@ -427,18 +478,18 @@ func (c *converter) syncIngress(ing *networking.Ingress) { uri = "/" } if host.FindPath(uri) != nil { - c.logger.Warn("skipping redeclared path '%s' of ingress '%s'", uri, fullIngName) + c.logger.Warn("skipping redeclared path '%s' of %v", uri, source) continue } svcName, svcPort, err := readServiceNamePort(&path.Backend) if err != nil { - c.logger.Warn("skipping backend config of ingress '%s': %v", fullIngName, err) + c.logger.Warn("skipping backend config of %v: %v", source, err) continue } fullSvcName := ing.Namespace + "/" + svcName backend, err := c.addBackendWithClass(source, hostname, uri, fullSvcName, svcPort, annBack, ingressClass) if err != nil { - c.logger.Warn("skipping backend config of ingress '%s': %v", fullIngName, err) + c.logger.Warn("skipping backend config of %v: %v", source, err) continue } match := c.readPathType(path, annHost[ingtypes.HostPathType]) @@ -476,9 +527,9 @@ func (c *converter) syncIngress(ing *networking.Ingress) { } else if host.TLS.TLSHash != tlsPath.SHA1Hash { msg := fmt.Sprintf("TLS of host '%s' was already assigned", host.Hostname) if tls.SecretName != "" { - c.logger.Warn("skipping TLS secret '%s' of ingress '%s': %s", tls.SecretName, fullIngName, msg) + c.logger.Warn("skipping TLS secret '%s' of %v: %s", tls.SecretName, source, msg) } else { - c.logger.Warn("skipping default TLS secret of ingress '%s': %s", fullIngName, msg) + c.logger.Warn("skipping default TLS secret of %v: %s", source, msg) } } } @@ -495,10 +546,92 @@ func (c *converter) syncIngress(ing *networking.Ingress) { if tlsAcme { if tls.SecretName != "" { secretName := ing.Namespace + "/" + tls.SecretName + ingName := ing.Namespace + "/" + ing.Name c.haproxy.AcmeData().Storages().Acquire(secretName).AddDomains(tls.Hosts) - c.tracker.TrackStorage(convtypes.IngressType, fullIngName, secretName) + c.tracker.TrackStorage(convtypes.IngressType, ingName, secretName) } else { - c.logger.Warn("skipping cert signer of ingress '%s': missing secret name", fullIngName) + c.logger.Warn("skipping cert signer of %v: missing secret name", source) + } + } + } +} + +func (c *converter) syncIngressTCP(ing *networking.Ingress, tcpServicePort int, annTCP, annBack map[string]string) { + source := &annotations.Source{ + Namespace: ing.Namespace, + Name: ing.Name, + Type: "ingress", + } + addIngressBackend := func(rawHostname string, ingressBackend *networking.IngressBackend) error { + hostname := normalizeHostname(rawHostname, tcpServicePort) + tcpService, err := c.addTCPService(source, hostname, tcpServicePort, annTCP) + if err != nil { + return err + } + defer func() { + if tcpService.Backend.IsEmpty() { + c.haproxy.TCPServices().RemoveService(hostname) + } + }() + svcName, svcPort, err := readServiceNamePort(ingressBackend) + if err != nil { + return err + } + if !tcpService.Backend.IsEmpty() { + return fmt.Errorf("service '%s' on %v: backend for port '%d' was already assinged", svcName, source, tcpServicePort) + } + fullSvcName := ing.Namespace + "/" + svcName + ingressClass := c.readIngressClass(source, hostname, ing.Spec.IngressClassName) + backend, err := c.addBackendWithClass(source, hostname, "/", fullSvcName, svcPort, annBack, ingressClass) + if err != nil { + return err + } + tcpService.Backend = backend.BackendID() + backend.ModeTCP = true + return nil + } + if ing.Spec.DefaultBackend != nil { + err := addIngressBackend("", ing.Spec.DefaultBackend) + if err != nil { + c.logger.Warn("skipping default backend on %v: %v", source, err) + } + } + for _, rule := range ing.Spec.Rules { + if rule.HTTP == nil { + continue + } + for _, path := range rule.HTTP.Paths { + if path.Path != "" && path.Path != "/" { + c.logger.Warn("skipping backend declaration on path '%s' of %v: tcp services do not support path", path.Path, source) + continue + } + err := addIngressBackend(rule.Host, &path.Backend) + if err != nil { + c.logger.Warn("skipping path declaration on %v: %v", source, err) + } + } + } + for _, tls := range ing.Spec.TLS { + // tls secret + for _, hostname := range tls.Hosts { + tcpPort := c.haproxy.TCPServices().FindTCPPort(tcpServicePort) + if tcpPort == nil { + c.logger.Warn("skipping TLS of tcp service on %v: backend was not configured", source) + continue + } + tlsPath := c.addTLS(source, normalizeHostname(hostname, tcpServicePort), tls.SecretName) + if tcpPort.TLS.TLSHash == "" { + tcpPort.TLS.TLSFilename = tlsPath.Filename + tcpPort.TLS.TLSHash = tlsPath.SHA1Hash + tcpPort.TLS.TLSCommonName = tlsPath.CommonName + tcpPort.TLS.TLSNotAfter = tlsPath.NotAfter + } else if tcpPort.TLS.TLSHash != tlsPath.SHA1Hash { + msg := fmt.Sprintf("TLS of tcp service port '%d' was already assigned", tcpServicePort) + if tls.SecretName != "" { + c.logger.Warn("skipping TLS secret '%s' of %v: %s", tls.SecretName, source, msg) + } else { + c.logger.Warn("skipping default TLS secret of %v: %s", source, msg) + } } } } @@ -516,8 +649,24 @@ func (c *converter) syncChangedEndpointCookies() { } } +func (c *converter) fullSyncTCP() { + for _, tcpPort := range c.haproxy.TCPServices().Items() { + if ann, found := c.tcpsvcAnnotations[tcpPort]; found { + c.updater.UpdateTCPPortConfig(tcpPort, ann) + tcpHost := tcpPort.DefaultHost() + if tcpHost != nil { + c.updater.UpdateTCPHostConfig(tcpHost, ann) + } + for _, tcpHost := range tcpPort.Hosts() { + c.updater.UpdateTCPHostConfig(tcpHost, ann) + } + } + } +} + func (c *converter) fullSyncAnnotations() { c.updater.UpdateGlobalConfig(c.haproxy, c.globalConfig) + c.fullSyncTCP() for _, host := range c.haproxy.Hosts().Items() { if ann, found := c.hostAnnotations[host]; found { c.updater.UpdateHostConfig(host, ann) @@ -531,6 +680,7 @@ func (c *converter) fullSyncAnnotations() { } func (c *converter) partialSyncAnnotations() { + c.fullSyncTCP() for _, host := range c.haproxy.Hosts().ItemsAdd() { if ann, found := c.hostAnnotations[host]; found { c.updater.UpdateHostConfig(host, ann) @@ -607,6 +757,25 @@ func (c *converter) addDefaultHostBackend(source *annotations.Source, fullSvcNam return nil } +func (c *converter) addTCPService(source *annotations.Source, hostname string, port int, ann map[string]string) (*hatypes.TCPServiceHost, error) { + tcpPort, tcpHost := c.haproxy.TCPServices().AcquireTCPService(hostname) + if !tcpHost.Backend.IsEmpty() { + tcpservice := strings.TrimPrefix(hostname, hatypes.DefaultHost) + return nil, fmt.Errorf("tcp service %s was already assigned to %s", tcpservice, tcpHost.Backend) + } + c.tracker.TrackHostname(convtypes.IngressType, source.FullName(), hostname) + mapper, found := c.tcpsvcAnnotations[tcpPort] + if !found { + mapper = c.mapBuilder.NewMapper() + c.tcpsvcAnnotations[tcpPort] = mapper + } + conflict := mapper.AddAnnotations(source, hatypes.CreatePathLink(hostname, "/"), ann) + if len(conflict) > 0 { + c.logger.Warn("skipping tcp service annotation(s) from %v due to conflict: %v", source, conflict) + } + return tcpHost, nil +} + func (c *converter) addHost(hostname string, source *annotations.Source, ann map[string]string) *hatypes.Host { // TODO build a stronger tracking host := c.haproxy.Hosts().AcquireHost(hostname) @@ -654,7 +823,7 @@ func (c *converter) addBackendWithClass(source *annotations.Source, hostname, ur if !found { // New backend, initialize with service annotations, giving precedence mapper = c.mapBuilder.NewMapper() - _, ann := c.readAnnotations(svc.Annotations) + _, _, ann := c.readAnnotations(svc.Annotations) mapper.AddAnnotations(&annotations.Source{ Namespace: namespace, Name: svcName, @@ -773,21 +942,24 @@ func (c *converter) addEndpoints(svc *api.Service, svcPort *api.ServicePort, bac return nil } -func (c *converter) readAnnotations(annotations map[string]string) (annHost, annBack map[string]string) { +func (c *converter) readAnnotations(annotations map[string]string) (annTCP, annHost, annBack map[string]string) { + annTCP = make(map[string]string, len(annotations)) annHost = make(map[string]string, len(annotations)) annBack = make(map[string]string, len(annotations)) prefix := c.options.AnnotationPrefix + "/" for annName, annValue := range annotations { if strings.HasPrefix(annName, prefix) { name := strings.TrimPrefix(annName, prefix) - if _, isHostAnn := ingtypes.AnnHost[name]; isHostAnn { + if _, isTCPAnn := ingtypes.AnnTCP[name]; isTCPAnn { + annTCP[name] = annValue + } else if _, isHostAnn := ingtypes.AnnHost[name]; isHostAnn { annHost[name] = annValue } else { annBack[name] = annValue } } } - return annHost, annBack + return annTCP, annHost, annBack } func (c *converter) readParameters(ingressClass *networking.IngressClass, trackingHostname string) map[string]string { diff --git a/pkg/converters/ingress/ingress_test.go b/pkg/converters/ingress/ingress_test.go index 6b697b40d..8e0425660 100644 --- a/pkg/converters/ingress/ingress_test.go +++ b/pkg/converters/ingress/ingress_test.go @@ -17,6 +17,7 @@ limitations under the License. package ingress import ( + "sort" "strconv" "strings" "testing" @@ -944,6 +945,15 @@ func TestSyncMultiNamespace(t *testing.T) { port: 8080` + defaultBackendConfig) } +func paramToMap(param ...string) map[string]string { + res := make(map[string]string, len(param)) + for _, p := range param { + v := strings.SplitN(p, "=", 2) + res[v[0]] = v[1] + } + return res +} + func TestSyncPartial(t *testing.T) { svcDefault := [][]string{ {"default/echo1", "8080", "172.17.0.11"}, @@ -1355,24 +1365,16 @@ WARN using default certificate due to an error reading secret 'default/tls1' on c.logger.Logging = []string{} ings := func(slice *[]*networking.Ingress, params [][]string) { - paramToMap := func(param []string) map[string]string { - res := make(map[string]string, len(param)) - for _, p := range param { - v := strings.SplitN(p, "=", 2) - res[v[0]] = v[1] - } - return res - } for _, param := range params { var ing *networking.Ingress switch len(param) { case 3: ing = c.createIng2(param[0], param[1]) - ing.SetAnnotations(paramToMap(param[2:])) + ing.SetAnnotations(paramToMap(param[2])) case 4: ing = c.createIng1(param[0], param[1], param[2], param[3]) case 5: - ing = c.createIng1Ann(param[0], param[1], param[2], param[3], paramToMap(param[4:])) + ing = c.createIng1Ann(param[0], param[1], param[2], param[3], paramToMap(param[4])) } *slice = append(*slice, ing) } @@ -1452,6 +1454,320 @@ func TestSyncPartialDefaultBackend(t *testing.T) { * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +func TestSyncTCPServicePort(t *testing.T) { + testCases := []struct { + ing [][]string + expect string + logging string + }{ + // 0 + { + expect: `[]`, + }, + // 1 + { + ing: [][]string{ + {"7001"}, + }, + expect: ` +- backends: [] + defaultbackend: default_echo1_8080 + port: 7001 + proxyprot: false + tls: {}`, + }, + // 2 + { + ing: [][]string{ + {"7001", "/", "echonotfound:8080"}, + }, + expect: `[]`, + logging: `WARN skipping path declaration on ingress 'default/echo1': service not found: 'default/echonotfound'`, + }, + // 3 + { + ing: [][]string{ + {"7001", "/", "echo1:notvalidport"}, + }, + expect: `[]`, + logging: `WARN skipping path declaration on ingress 'default/echo1': port not found: 'notvalidport'`, + }, + // 4 + { + ing: [][]string{ + {"7001", "", "echo1:8080"}, + }, + expect: ` +- backends: [] + defaultbackend: default_echo1_8080 + port: 7001 + proxyprot: false + tls: {}`, + }, + // 5 + { + ing: [][]string{ + {"7001", "/", "echo1:8080"}, + {"7001", "/", "echo1:8080"}, + }, + expect: ` +- backends: [] + defaultbackend: default_echo1_8080 + port: 7001 + proxyprot: false + tls: {}`, + logging: `WARN skipping path declaration on ingress 'default/echo2': tcp service :7001 was already assigned to default_echo1_8080`, + }, + // 6 + { + ing: [][]string{ + {"7001", "/", "echo1:8080", "tls1"}, + }, + expect: ` +- backends: [] + defaultbackend: default_echo1_8080 + port: 7001 + proxyprot: false + tls: + tlsfilename: /tls/default/tls1.pem`, + }, + // 7 + { + ing: [][]string{ + {"7001", "/", "echo1:8080", "tls-invalid"}, + }, + expect: ` +- backends: [] + defaultbackend: default_echo1_8080 + port: 7001 + proxyprot: false + tls: + tlsfilename: /tls/tls-default.pem`, + logging: `WARN using default certificate due to an error reading secret 'tls-invalid' on ingress 'default/echo1': secret not found: 'default/tls-invalid'`, + }, + // 8 + { + ing: [][]string{ + {"7001", "tls1"}, + }, + expect: `[]`, + logging: `WARN skipping TLS of tcp service on ingress 'default/echo1': backend was not configured`, + }, + // 9 + { + ing: [][]string{ + {"7001", "/", "echo1:8080", "tls1"}, + {"7001", "/", "echo1:8080", "tls1"}, + }, + expect: ` +- backends: [] + defaultbackend: default_echo1_8080 + port: 7001 + proxyprot: false + tls: + tlsfilename: /tls/default/tls1.pem`, + logging: `WARN skipping path declaration on ingress 'default/echo2': tcp service :7001 was already assigned to default_echo1_8080`, + }, + // 10 + { + ing: [][]string{ + {"7001", "/", "echo1:8080", "tls1"}, + {"7001", "/", "echo1:8080", "tls2"}, + }, + expect: ` +- backends: [] + defaultbackend: default_echo1_8080 + port: 7001 + proxyprot: false + tls: + tlsfilename: /tls/default/tls1.pem`, + logging: ` +WARN skipping path declaration on ingress 'default/echo2': tcp service :7001 was already assigned to default_echo1_8080 +WARN skipping TLS secret 'tls2' of ingress 'default/echo2': TLS of tcp service port '7001' was already assigned`, + }, + // 11 + { + ing: [][]string{ + {"7001", "/", "echo1:8080"}, + {"7001", "tls1"}, + }, + expect: ` +- backends: [] + defaultbackend: default_echo1_8080 + port: 7001 + proxyprot: false + tls: + tlsfilename: /tls/default/tls1.pem`, + }, + // 12 + { + ing: [][]string{ + {"echo.local:7001", "/", "echo1:8080,echo2:8080"}, + }, + expect: ` +- backends: + - default_echo1_8080 + defaultbackend: "" + port: 7001 + proxyprot: false + tls: {}`, + logging: `WARN skipping path declaration on ingress 'default/echo1': tcp service echo.local:7001 was already assigned to default_echo1_8080`, + }, + // 13 + { + ing: [][]string{ + {"echo1.local:7001", "/", "echo1:8080"}, + {"echo2.local:7001", "/", "echo2:8080"}, + }, + expect: ` +- backends: + - default_echo1_8080 + - default_echo2_8080 + defaultbackend: "" + port: 7001 + proxyprot: false + tls: {}`, + }, + // 14 + { + ing: [][]string{ + {"echo1.local:7001", "/", "echo1:8080"}, + {"echo2.local:7001", "/", "echo2:8080"}, + {"echo2.local:7001", "/", "echo1:8080"}, + }, + expect: ` +- backends: + - default_echo1_8080 + - default_echo2_8080 + defaultbackend: "" + port: 7001 + proxyprot: false + tls: {}`, + logging: `WARN skipping path declaration on ingress 'default/echo3': tcp service echo2.local:7001 was already assigned to default_echo2_8080`, + }, + // 15 + { + ing: [][]string{ + {":7001", "/", "echo1:8080"}, + {"echo1.local:7001", "/", "echo1:8080"}, + {"echo2.local:7001", "/", "echo2:8080"}, + }, + expect: ` +- backends: + - default_echo1_8080 + - default_echo2_8080 + defaultbackend: default_echo1_8080 + port: 7001 + proxyprot: false + tls: {}`, + }, + // 16 + { + ing: [][]string{ + {"7001", "/app", "echo1:8080"}, + }, + expect: `[]`, + logging: `WARN skipping backend declaration on path '/app' of ingress 'default/echo1': tcp services do not support path`, + }, + // 17 + { + ing: [][]string{ + {"7001", "/", "echo1:8080"}, + {"7002", "/", "echo1:8080", "tls2", "ingress.kubernetes.io/" + ingtypes.TCPTCPServiceProxyProto + "=true"}, + {"7003", "/", "echo2:8080"}, + }, + expect: ` +- backends: [] + defaultbackend: default_echo1_8080 + port: 7001 + proxyprot: false + tls: {} +- backends: [] + defaultbackend: default_echo1_8080 + port: 7002 + proxyprot: true + tls: + tlsfilename: /tls/default/tls2.pem +- backends: [] + defaultbackend: default_echo2_8080 + port: 7003 + proxyprot: false + tls: {}`, + }, + } + for i, test := range testCases { + c := setup(t) + + c.createSvc1("default/echo1", "8080", "172.17.0.11") + c.createSvc1("default/echo2", "8080", "172.17.0.12") + c.createSecretTLS1("default/tls1") + c.createSecretTLS1("default/tls2") + + for _, params := range test.ing { + n := strconv.Itoa(len(c.cache.IngList) + 1) + name := "default/echo" + n + domain := "" + port := params[0] + if pos := strings.Index(port, ":"); pos >= 0 { + domain = port[:pos] + port = port[pos+1:] + } + annPort := "ingress.kubernetes.io/" + ingtypes.TCPTCPServicePort + "=" + port + var ing *networking.Ingress + switch len(params) { + case 1: + ing = c.createIng2(name, "echo1:8080") + ing.SetAnnotations(paramToMap(annPort)) + case 2: + ing = c.createIngTLS1(name, domain, "/", ":", params[1]) + ing.Spec.Rules = nil + ing.SetAnnotations(paramToMap(annPort)) + case 3: + ssvc := strings.Split(params[2], ",") // two services (hence paths) in the same ing.Spec.Rules[*].HTTP + ing = c.createIng1Ann(name, domain, params[1], ssvc[0], paramToMap(annPort)) + for _, svc := range ssvc[1:] { + // TODO migrate to an ingress constructor + http := ing.Spec.Rules[0].HTTP + path := networking.HTTPIngressPath{ + Path: params[1], // + "/" + strconv.Itoa(len(http.Paths)), + } + s := strings.Split(svc, ":") + path.Backend.Service = &networking.IngressServiceBackend{ + Name: s[0], + Port: createServicePort(s[1]), + } + http.Paths = append(http.Paths, path) + } + case 4: + ing = c.createIngTLS1(name, domain, params[1], params[2], params[3]) + ing.SetAnnotations(paramToMap(annPort)) + case 5: + ing = c.createIngTLS1(name, domain, params[1], params[2], params[3]) + ing.SetAnnotations(paramToMap(annPort, params[4])) + default: + panic("invalid size") + } + c.cache.IngList = append(c.cache.IngList, ing) + } + c.Sync() + + c.compareConfigFront("[]") + for _, tcp := range c.hconfig.TCPServices().BuildSortedItems() { + for _, host := range tcp.Hosts() { + backend := c.hconfig.Backends().FindBackendID(host.Backend) + if !backend.ModeTCP { + t.Errorf("mode tcp in %d, backend %s, expected true but was false", i, backend.BackendID()) + } + + } + } + c.compareConfigTCPService(test.expect) + c.logger.CompareLogging(test.logging) + + c.teardown() + } +} + func TestSyncAnnFront(t *testing.T) { c := setup(t) defer c.teardown() @@ -2098,6 +2414,13 @@ type updaterMock struct{} func (u *updaterMock) UpdateGlobalConfig(haproxyConfig haproxy.Config, config *annotations.Mapper) { } +func (u *updaterMock) UpdateTCPPortConfig(tcp *hatypes.TCPServicePort, mapper *annotations.Mapper) { + tcp.ProxyProt = mapper.Get(ingtypes.TCPTCPServiceProxyProto).Bool() +} + +func (u *updaterMock) UpdateTCPHostConfig(tcp *hatypes.TCPServiceHost, mapper *annotations.Mapper) { +} + func (u *updaterMock) UpdateHostConfig(host *hatypes.Host, mapper *annotations.Mapper) { host.RootRedirect = mapper.Get(ingtypes.HostAppRoot).Value } @@ -2107,6 +2430,46 @@ func (u *updaterMock) UpdateBackendConfig(backend *hatypes.Backend, mapper *anno backend.BalanceAlgorithm = mapper.Get(ingtypes.BackBalanceAlgorithm).Value } +type ( + tcpServiceMock struct { + Backends []string + DefaultBackend string + Port int + ProxyProt bool + TLS tlsMock + } +) + +func convertTCPService(hatcpserviceports ...*hatypes.TCPServicePort) []tcpServiceMock { + tcpServices := []tcpServiceMock{} + for _, hasvc := range hatcpserviceports { + var backends []string + for _, h := range hasvc.Hosts() { + backends = append(backends, h.Backend.String()) + } + sort.Strings(backends) + var defaultBackend string + if hasvc.DefaultHost() != nil { + defaultBackend = hasvc.DefaultHost().Backend.String() + } + svc := tcpServiceMock{ + Backends: backends, + DefaultBackend: defaultBackend, + Port: hasvc.Port(), + ProxyProt: hasvc.ProxyProt, + TLS: tlsMock{ + TLSFilename: hasvc.TLS.TLSFilename, + }, + } + tcpServices = append(tcpServices, svc) + } + return tcpServices +} + +func (c *testConfig) compareConfigTCPService(expected string) { + c.compareText(_yamlMarshal(convertTCPService(c.hconfig.TCPServices().BuildSortedItems()...)), expected) +} + type ( pathMock struct { Path string diff --git a/pkg/converters/ingress/types/annotations.go b/pkg/converters/ingress/types/annotations.go index a276c22dd..0ffa9cf5e 100644 --- a/pkg/converters/ingress/types/annotations.go +++ b/pkg/converters/ingress/types/annotations.go @@ -16,6 +16,23 @@ limitations under the License. package types +const ( + TCPConfigTCPService = "config-tcp-service" + TCPTCPServiceLogFormat = "tcp-service-log-format" + TCPTCPServicePort = "tcp-service-port" + TCPTCPServiceProxyProto = "tcp-service-proxy-protool" +) + +var ( + // AnnTCP ... + AnnTCP = map[string]struct{}{ + TCPConfigTCPService: {}, + TCPTCPServiceLogFormat: {}, + TCPTCPServicePort: {}, + TCPTCPServiceProxyProto: {}, + } +) + // Host Annotations const ( HostAppRoot = "app-root" diff --git a/pkg/haproxy/config.go b/pkg/haproxy/config.go index e15a3411f..7f2199cfb 100644 --- a/pkg/haproxy/config.go +++ b/pkg/haproxy/config.go @@ -32,11 +32,13 @@ import ( type Config interface { Frontend() *hatypes.Frontend SyncConfig() + WriteTCPServicesMaps() error WriteFrontendMaps() error WriteBackendMaps() error AcmeData() *hatypes.AcmeData Global() *hatypes.Global TCPBackends() *hatypes.TCPBackends + TCPServices() *hatypes.TCPServices Hosts() *hatypes.Hosts Backends() *hatypes.Backends Userlists() *hatypes.Userlists @@ -56,6 +58,7 @@ type config struct { hosts *hatypes.Hosts backends *hatypes.Backends tcpbackends *hatypes.TCPBackends + tcpservices *hatypes.TCPServices userlists *hatypes.Userlists } @@ -77,6 +80,7 @@ func createConfig(options options) *config { hosts: hatypes.CreateHosts(), backends: hatypes.CreateBackends(options.shardCount), tcpbackends: hatypes.CreateTCPBackends(), + tcpservices: hatypes.CreateTCPServices(), userlists: hatypes.CreateUserlists(), } } @@ -136,6 +140,26 @@ func (c *config) SyncConfig() { } } +// WriteTCPServicesMaps reads the model and writes haproxy's maps +// used in the tcp services. Should be called before write the main +// config file. This func doesn't change model state, except the +// link to the tcp services maps. +func (c *config) WriteTCPServicesMaps() error { + if !c.tcpservices.Changed() { + return nil + } + mapBuilder := hatypes.CreateMaps(c.global.MatchOrder) + for _, tcpPort := range c.tcpservices.Items() { + sniMap := mapBuilder.AddMap(fmt.Sprintf("%s/_tcp_sni_%d.map", c.options.mapsDir, tcpPort.Port())) + for _, tcpHost := range tcpPort.BuildSortedItems() { + sniMap.AddHostnameMapping(tcpHost.Hostname(), tcpHost.Backend.String()) + } + tcpPort.SNIMap = sniMap + } + err := writeMaps(mapBuilder, c.options.mapsTemplate) + return err +} + // WriteFrontendMaps reads the model and writes haproxy's maps // used in the frontend. Should be called before write the main // config file. This func doesn't change model state, except the @@ -362,6 +386,10 @@ func (c *config) TCPBackends() *hatypes.TCPBackends { return c.tcpbackends } +func (c *config) TCPServices() *hatypes.TCPServices { + return c.tcpservices +} + func (c *config) Hosts() *hatypes.Hosts { return c.hosts } @@ -397,6 +425,7 @@ func (c *config) Commit() { c.hosts.Commit() c.backends.Commit() c.tcpbackends.Commit() + c.tcpservices.Commit() c.userlists.Commit() c.acmeData.Storages().Commit() } diff --git a/pkg/haproxy/dynupdate.go b/pkg/haproxy/dynupdate.go index 995041333..1bdf01dc6 100644 --- a/pkg/haproxy/dynupdate.go +++ b/pkg/haproxy/dynupdate.go @@ -85,6 +85,9 @@ func (d *dynUpdater) checkConfigChange() bool { diff = append(diff, "global") } if d.config.tcpbackends.Changed() { + diff = append(diff, "tcp-services (configmap)") + } + if d.config.tcpservices.Changed() { diff = append(diff, "tcp-services") } if d.config.frontend.Changed() { diff --git a/pkg/haproxy/instance.go b/pkg/haproxy/instance.go index a2e2c818f..6a48ef6e8 100644 --- a/pkg/haproxy/instance.go +++ b/pkg/haproxy/instance.go @@ -243,6 +243,11 @@ func (i *instance) haproxyUpdate(timer *utils.Timer) { defer i.config.Commit() i.config.SyncConfig() i.config.Shrink() + if err := i.config.WriteTCPServicesMaps(); err != nil { + i.logger.Error("error building tcp services maps: %v", err) + i.metrics.IncUpdateNoop() + return + } if err := i.config.WriteFrontendMaps(); err != nil { i.logger.Error("error building frontend maps: %v", err) i.metrics.IncUpdateNoop() diff --git a/pkg/haproxy/instance_test.go b/pkg/haproxy/instance_test.go index ce3eb1708..0fc54b0dd 100644 --- a/pkg/haproxy/instance_test.go +++ b/pkg/haproxy/instance_test.go @@ -1591,6 +1591,242 @@ frontend _front_https } } +func TestInstanceTCPServices(t *testing.T) { + c := setup(t) + defer c.teardown() + + var h *hatypes.Host + var b *hatypes.Backend + + b = c.config.Backends().AcquireBackend("d1", "app", "8080") + b.Endpoints = []*hatypes.Endpoint{endpointS1} + h = c.config.Hosts().AcquireHost("d1.local") + h.AddPath(b, "/", hatypes.MatchBegin) + + b2 := c.config.Backends().AcquireBackend("d2", "app", "8080") + b2.Endpoints = []*hatypes.Endpoint{endpointS21, endpointS22} + b3 := c.config.Backends().AcquireBackend("d3", "app", "8080") + b3.Endpoints = []*hatypes.Endpoint{endpointS31, endpointS32} + + services := []struct { + port int + hostname string + backend hatypes.BackendID + proxyProt bool + tls hatypes.TLSConfig + custom []string + }{ + { + port: 7000, + }, + { + port: 7001, + backend: b.BackendID(), + }, + { + port: 7002, + backend: b.BackendID(), + proxyProt: true, + }, + { + port: 7003, + backend: b.BackendID(), + proxyProt: true, + tls: hatypes.TLSConfig{ + TLSFilename: "/ssl/7003.pem", + }, + }, + { + port: 7004, + backend: b.BackendID(), + tls: hatypes.TLSConfig{ + TLSFilename: "/ssl/7004.pem", + }, + }, + { + port: 7005, + backend: b.BackendID(), + tls: hatypes.TLSConfig{ + TLSFilename: "/ssl/7005.pem", + CAFilename: "/ssl/ca-7005.pem", + }, + }, + { + port: 7006, + backend: b.BackendID(), + tls: hatypes.TLSConfig{ + TLSFilename: "/ssl/7006.pem", + CAFilename: "/ssl/ca-7006.pem", + CRLFilename: "/ssl/crl-7006.pem", + }, + }, + { + port: 7007, + backend: b.BackendID(), + tls: hatypes.TLSConfig{ + ALPN: "h2,http/1.1", + TLSFilename: "/ssl/7007.pem", + }, + }, + { + port: 7008, + backend: b.BackendID(), + tls: hatypes.TLSConfig{ + TLSFilename: "/ssl/7008.pem", + CAFilename: "/ssl/ca-7008.pem", + CAVerifyOptional: true, + }, + }, + { + port: 7009, + backend: b.BackendID(), + tls: hatypes.TLSConfig{ + TLSFilename: "/ssl/7009.pem", + Ciphers: "ECDHE-ECDSA-AES128-GCM-SHA256", + CipherSuites: "TLS_AES_128_GCM_SHA256", + }, + }, + { + port: 7010, + backend: b.BackendID(), + tls: hatypes.TLSConfig{ + TLSFilename: "/ssl/7010.pem", + Options: "force-tlsv13", + }, + }, + { + port: 7011, + backend: b.BackendID(), + }, + { + port: 7011, + hostname: "local2", + backend: b2.BackendID(), + }, + { + port: 7011, + hostname: "local3", + backend: b3.BackendID(), + }, + { + port: 7011, + hostname: "*.local4", + backend: b3.BackendID(), + }, + { + port: 7012, + backend: b.BackendID(), + custom: []string{"## custom for TCP 7012"}, + }, + { + port: 7013, + backend: b.BackendID(), + custom: []string{"## custom for TCP 7013", "## multi line"}, + }, + } + + for _, svc := range services { + hostname := svc.hostname + if hostname == "" { + hostname = hatypes.DefaultHost + } + p, h := c.config.TCPServices().AcquireTCPService(fmt.Sprintf("%s:%d", hostname, svc.port)) + p.ProxyProt = svc.proxyProt + p.TLS = svc.tls + p.CustomConfig = svc.custom + h.Backend = svc.backend + } + + c.Update() + c.checkConfig(` +<> +<> +backend d1_app_8080 + mode http + server s1 172.17.0.11:8080 weight 100 +backend d2_app_8080 + mode http + server s21 172.17.0.121:8080 weight 100 + server s22 172.17.0.122:8080 weight 100 +backend d3_app_8080 + mode http + server s31 172.17.0.131:8080 weight 100 + server s32 172.17.0.132:8080 weight 100 +<> +frontend _front_tcp_7000 + bind :7000 + mode tcp +frontend _front_tcp_7001 + bind :7001 + mode tcp + default_backend d1_app_8080 +frontend _front_tcp_7002 + bind :7002 accept-proxy + mode tcp + default_backend d1_app_8080 +frontend _front_tcp_7003 + bind :7003 accept-proxy ssl crt /ssl/7003.pem + mode tcp + default_backend d1_app_8080 +frontend _front_tcp_7004 + bind :7004 ssl crt /ssl/7004.pem + mode tcp + default_backend d1_app_8080 +frontend _front_tcp_7005 + bind :7005 ssl crt /ssl/7005.pem ca-file /ssl/ca-7005.pem verify required + mode tcp + default_backend d1_app_8080 +frontend _front_tcp_7006 + bind :7006 ssl crt /ssl/7006.pem ca-file /ssl/ca-7006.pem verify required crl-file /ssl/crl-7006.pem + mode tcp + default_backend d1_app_8080 +frontend _front_tcp_7007 + bind :7007 ssl crt /ssl/7007.pem alpn h2,http/1.1 + mode tcp + default_backend d1_app_8080 +frontend _front_tcp_7008 + bind :7008 ssl crt /ssl/7008.pem ca-file /ssl/ca-7008.pem verify optional + mode tcp + default_backend d1_app_8080 +frontend _front_tcp_7009 + bind :7009 ssl crt /ssl/7009.pem ciphers ECDHE-ECDSA-AES128-GCM-SHA256 ciphersuites TLS_AES_128_GCM_SHA256 + mode tcp + default_backend d1_app_8080 +frontend _front_tcp_7010 + bind :7010 ssl crt /ssl/7010.pem force-tlsv13 + mode tcp + default_backend d1_app_8080 +frontend _front_tcp_7011 + bind :7011 + mode tcp + tcp-request inspect-delay 5s + tcp-request content set-var(req.tcpback) req.ssl_sni,lower,map_str(/etc/haproxy/maps/_tcp_sni_7011__exact.map) + tcp-request content set-var(req.tcpback) req.ssl_sni,lower,map_reg(/etc/haproxy/maps/_tcp_sni_7011__regex.map) if !{ var(req.tcpback) -m found } + tcp-request content accept if { req.ssl_hello_type 1 } + use_backend %[var(req.tcpback)] if { var(req.tcpback) -m found } + default_backend d1_app_8080 +frontend _front_tcp_7012 + bind :7012 + mode tcp + ## custom for TCP 7012 + default_backend d1_app_8080 +frontend _front_tcp_7013 + bind :7013 + mode tcp + ## custom for TCP 7013 + ## multi line + default_backend d1_app_8080 +<> +<> +`) + c.checkMap("_tcp_sni_7011__exact.map", ` +local2 d2_app_8080 +local3 d3_app_8080`) + c.checkMap("_tcp_sni_7011__regex.map", ` +^[^.]+\.local4$ d3_app_8080`) + c.logger.CompareLogging(defaultLogging) +} + func TestInstanceTCPBackend(t *testing.T) { testCases := []struct { doconfig func(c *testConfig) @@ -2484,6 +2720,9 @@ func TestInstanceCustomProxy(t *testing.T) { tcp := c.config.tcpbackends.Acquire("default_pgsql", 5432) tcp.AddEndpoint("172.17.0.21", 5432) + _, tcpHost := c.config.tcpservices.AcquireTCPService(hatypes.DefaultHost + ":7001") + tcpHost.Backend = b.BackendID() + c.config.Global().CustomProxy = map[string][]string{ "missing": {"## comment"}, "_tcp_default_pgsql_5432": {"## custom for _tcp_default_pgsql_5432"}, @@ -2493,6 +2732,7 @@ func TestInstanceCustomProxy(t *testing.T) { "_error404": {"## custom for _error404", "## line 2"}, "_auth_4001": {"## custom for _auth_4001"}, "_front__auth": {"## custom for _front__auth"}, + "_front_tcp_7001": {"## custom for _front_tcp_7001"}, "_front__tls": {"## custom for _front__tls"}, "_front_http": {"## custom for _front_http"}, "_front_https": {"## custom for _front_https"}, @@ -2535,6 +2775,11 @@ frontend _front__auth bind 127.0.0.1:4001 ## custom for _front__auth use_backend _auth_backend001_5000 +frontend _front_tcp_7001 + bind :7001 + mode tcp + ## custom for _front_tcp_7001 + default_backend d1_app_8080 listen _front__tls mode tcp bind :443 @@ -3025,6 +3270,14 @@ func TestInstanceSyslog(t *testing.T) { h = c.config.Hosts().AcquireHost("d1.local") h.AddPath(b, "/", hatypes.MatchBegin) + tcpPort1, tcpHost1 := c.config.tcpservices.AcquireTCPService(":7001") + tcpPort1.LogFormat = "default" + tcpHost1.Backend = b.BackendID() + + tcpPort2, tcpHost2 := c.config.tcpservices.AcquireTCPService(":7002") + tcpPort2.LogFormat = "%[src]" + tcpHost2.Backend = b.BackendID() + syslog := &c.config.Global().Syslog syslog.Endpoint = "127.0.0.1:1514" syslog.Format = "rfc3164" @@ -3055,6 +3308,16 @@ backend d1_app_8080 mode http server s1 172.17.0.11:8080 weight 100 <> +frontend _front_tcp_7001 + bind :7001 + mode tcp + option tcplog + default_backend d1_app_8080 +frontend _front_tcp_7002 + bind :7002 + mode tcp + log-format %[src] + default_backend d1_app_8080 frontend _front_http mode http bind :80 diff --git a/pkg/haproxy/types/backends.go b/pkg/haproxy/types/backends.go index 669f8fd93..2df1cf0f3 100644 --- a/pkg/haproxy/types/backends.go +++ b/pkg/haproxy/types/backends.go @@ -270,6 +270,11 @@ func (b *Backends) RemoveAll(backendID []BackendID) { } } +// IsEmpty ... +func (b BackendID) IsEmpty() bool { + return b.Name == "" +} + func (b BackendID) String() string { if b.id == "" { b.id = buildID(b.Namespace, b.Name, b.Port) diff --git a/pkg/haproxy/types/tcpservices.go b/pkg/haproxy/types/tcpservices.go new file mode 100644 index 000000000..b5ae9cb34 --- /dev/null +++ b/pkg/haproxy/types/tcpservices.go @@ -0,0 +1,185 @@ +/* +Copyright 2021 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "sort" + "strconv" + "strings" +) + +// CreateTCPServices ... +func CreateTCPServices() *TCPServices { + return &TCPServices{ + items: map[int]*TCPServicePort{}, + } +} + +// AcquireTCPService ... +func (s *TCPServices) AcquireTCPService(service string) (*TCPServicePort, *TCPServiceHost) { + hostname, port := splitService(service) + tcpPort := s.acquireTCPPort(port) + tcpHost, found := tcpPort.acquireHost(hostname) + if !found { + s.changed = true + } + return tcpPort, tcpHost +} + +func (s *TCPServices) acquireTCPPort(port int) *TCPServicePort { + tcpPort, found := s.items[port] + if !found { + tcpPort = &TCPServicePort{ + port: port, + hosts: map[string]*TCPServiceHost{}, + } + s.items[port] = tcpPort + s.changed = true + } + return tcpPort +} + +// FindTCPPort ... +func (s *TCPServices) FindTCPPort(port int) *TCPServicePort { + return s.items[port] +} + +// Items ... +func (s *TCPServices) Items() map[int]*TCPServicePort { + return s.items +} + +// BuildSortedItems ... +func (s *TCPServices) BuildSortedItems() []*TCPServicePort { + items := make([]*TCPServicePort, 0, len(s.items)) + for _, item := range s.items { + items = append(items, item) + } + if len(items) == 0 { + return nil + } + sort.Slice(items, func(i, j int) bool { + return items[i].port < items[j].port + }) + return items +} + +// The convention is to name tcp services as domain:port, all TCPServices receive +// service name or hostname in this format. This convention is mostly used by +// hostname tracking which is a ingress converter feature. Such convention and +// tracking stuff shouldn't be reflecting here. Time to use a proper type without +// conventions and assumptions. +// TODO Use a proper service name or hostname type +func splitService(service string) (hostname string, port int) { + hostname = service + if pos := strings.Index(hostname, ":"); pos >= 0 { + hostname = service[:pos] + port, _ = strconv.Atoi(service[pos+1:]) + } + return hostname, port +} + +// RemoveService ... +func (s *TCPServices) RemoveService(service string) { + hostname, port := splitService(service) + if item, found := s.items[port]; found { + if _, hasHost := item.hosts[hostname]; hasHost { + delete(item.hosts, hostname) + s.changed = true + } + if hostname == DefaultHost { + item.defaultHost = nil + s.changed = true + } + if item.isEmpty() { + delete(s.items, port) + s.changed = true + } + } +} + +// RemoveAll removes services declared as a slice of : +func (s *TCPServices) RemoveAll(services []string) { + for _, svc := range services { + s.RemoveService(svc) + } +} + +// Changed ... +func (s *TCPServices) Changed() bool { + return s.changed +} + +// Commit ... +func (s *TCPServices) Commit() { + s.changed = false +} + +func (s *TCPServicePort) isEmpty() bool { + return s.defaultHost == nil && len(s.hosts) == 0 +} + +func (s *TCPServicePort) acquireHost(hostname string) (tcpHost *TCPServiceHost, found bool) { + if hostname == DefaultHost && s.defaultHost != nil { + return s.defaultHost, true + } + tcpHost, found = s.hosts[hostname] + if !found { + tcpHost = &TCPServiceHost{hostname: hostname} + if hostname == DefaultHost { + s.defaultHost = tcpHost + } else { + s.hosts[hostname] = tcpHost + } + } + return tcpHost, found +} + +// Port ... +func (s *TCPServicePort) Port() int { + return s.port +} + +// Hosts ... +func (s *TCPServicePort) Hosts() map[string]*TCPServiceHost { + return s.hosts +} + +// BuildSortedItems ... +func (s *TCPServicePort) BuildSortedItems() []*TCPServiceHost { + items := make([]*TCPServiceHost, 0, len(s.hosts)) + for _, item := range s.hosts { + items = append(items, item) + } + if len(items) == 0 { + return nil + } + sort.Slice(items, func(i, j int) bool { + return items[i].hostname < items[j].hostname + }) + return items +} + +// DefaultHost ... +func (s *TCPServicePort) DefaultHost() *TCPServiceHost { + return s.defaultHost +} + +// Hostname ... +func (s *TCPServiceHost) Hostname() string { + return s.hostname +} diff --git a/pkg/haproxy/types/tcpservices_test.go b/pkg/haproxy/types/tcpservices_test.go new file mode 100644 index 000000000..ebda8dda9 --- /dev/null +++ b/pkg/haproxy/types/tcpservices_test.go @@ -0,0 +1,147 @@ +/* +Copyright 2021 The HAProxy Ingress Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "fmt" + "sort" + "testing" +) + +func TestRemoveAll(t *testing.T) { + testCases := []struct { + input []string + remove []string + expItems []string + expDefault []string + changed bool + }{ + // 0 + { + remove: []string{"local:7001"}, + }, + // 1 + { + input: []string{":7001", ":7002"}, + remove: []string{"local:7001"}, + expItems: []string{":7001", ":7002"}, + }, + // 2 + { + input: []string{":7001", ":7002"}, + remove: []string{"local"}, + expItems: []string{":7001", ":7002"}, + }, + // 3 + { + input: []string{":7001", ":7002"}, + remove: []string{"local:"}, + expItems: []string{":7001", ":7002"}, + }, + // 4 + { + input: []string{":7001", ":7002"}, + remove: []string{"7001"}, + expItems: []string{":7001", ":7002"}, + }, + // 5 + { + input: []string{":7001", ":7002"}, + remove: []string{":7002"}, + expItems: []string{":7001"}, + changed: true, + }, + // 6 + { + input: []string{":7001", ":7002"}, + remove: []string{":7003"}, + expItems: []string{":7001", ":7002"}, + }, + // 7 + { + input: []string{":7001", ":7002", ":7003"}, + remove: []string{":7001", ":7002"}, + expItems: []string{":7003"}, + changed: true, + }, + // 8 + { + input: []string{":7001"}, + remove: []string{":7001"}, + expItems: nil, + changed: true, + }, + // 9 + { + input: []string{"local1:7001", "local2:7001", "local3:7001"}, + remove: []string{"local1:7001"}, + expItems: []string{"local2:7001", "local3:7001"}, + changed: true, + }, + // 10 + { + input: []string{"local1:7001"}, + remove: []string{"local1:7002"}, + expItems: []string{"local1:7001"}, + }, + // 11 + { + input: []string{"local1:7001"}, + remove: []string{"local2:7001"}, + expItems: []string{"local1:7001"}, + }, + // 12 + { + input: []string{":7001"}, + remove: []string{}, + expItems: nil, + expDefault: []string{""}, + }, + // 13 + { + input: []string{":7001"}, + remove: []string{":7001"}, + expItems: nil, + expDefault: nil, + changed: true, + }, + } + for i, test := range testCases { + c := setup(t) + f := CreateTCPServices() + for _, input := range test.input { + f.AcquireTCPService(input) + } + f.changed = false + f.RemoveAll(test.remove) + var expItems, expDefault []string + for _, tcpPort := range f.Items() { + for _, tcpHost := range tcpPort.hosts { + expItems = append(expItems, fmt.Sprintf("%s:%d", tcpHost.hostname, tcpPort.port)) + } + if tcpPort.defaultHost != nil { + expDefault = append(expDefault, tcpPort.defaultHost.hostname) + } + } + sort.Strings(expItems) + sort.Strings(expDefault) + c.compareObjects("tcpservices items", i, expItems, test.expItems) + c.compareObjects("tcpservices default", i, expDefault, test.expDefault) + c.compareObjects("tcpservices changed", i, f.changed, test.changed) + c.teardown() + } +} diff --git a/pkg/haproxy/types/types.go b/pkg/haproxy/types/types.go index 29ace0a71..de11a6fcb 100644 --- a/pkg/haproxy/types/types.go +++ b/pkg/haproxy/types/types.go @@ -238,6 +238,48 @@ type ModSecurityTimeoutConfig struct { Processing string } +// TCPServices ... +type TCPServices struct { + items map[int]*TCPServicePort + changed bool +} + +// TCPServicePort ... +type TCPServicePort struct { + port int + hosts map[string]*TCPServiceHost + defaultHost *TCPServiceHost + CustomConfig []string + LogFormat string + ProxyProt bool + TLS TLSConfig + // + SNIMap *HostsMap +} + +// TCPServiceHost ... +type TCPServiceHost struct { + hostname string + Backend BackendID +} + +// TLSConfig ... +type TLSConfig struct { + ALPN string + CAFilename string + CAHash string + CAVerifyOptional bool + Ciphers string + CipherSuites string + CRLFilename string + CRLHash string + Options string + TLSCommonName string + TLSFilename string + TLSHash string + TLSNotAfter time.Time +} + // TCPBackends ... type TCPBackends struct { items, itemsAdd, itemsDel map[int]*TCPBackend @@ -455,20 +497,8 @@ type HostRedirectConfig struct { // HostTLSConfig ... type HostTLSConfig struct { - ALPN string - CAErrorPage string - CAFilename string - CAHash string - CAVerifyOptional bool - Ciphers string - CipherSuites string - CRLFilename string - CRLHash string - Options string - TLSCommonName string - TLSFilename string - TLSHash string - TLSNotAfter time.Time + TLSConfig + CAErrorPage string } // EndpointNaming ... diff --git a/rootfs/etc/templates/haproxy/haproxy.tmpl b/rootfs/etc/templates/haproxy/haproxy.tmpl index 1871ccbbb..7944d0639 100644 --- a/rootfs/etc/templates/haproxy/haproxy.tmpl +++ b/rootfs/etc/templates/haproxy/haproxy.tmpl @@ -20,7 +20,8 @@ {{- $cfg := .Cfg }} {{- $global := $cfg.Global }} {{- $userlists := $cfg.Userlists.BuildSortedItems }} - {{- $tcpbackends := $cfg.TCPBackends.BuildSortedItems}} + {{- $tcpbackends := $cfg.TCPBackends.BuildSortedItems }} + {{- $tcpservices := $cfg.TCPServices.BuildSortedItems }} {{- $backends := $cfg.Backends }} {{- $backendItems := $backends.BuildSortedItems }} {{- $frontend := $cfg.Frontend }} @@ -43,9 +44,7 @@ {{- template "backends" map $global $backendItems true }} {{- end }} {{- template "backend-support" map $global $hosts $backends }} - {{- if $fmaps }} - {{- template "frontends" map $global $frontend $hosts $fmaps $backends.DefaultBackend }} - {{- end }} + {{- template "frontends" map $global $frontend $hosts $fmaps $backends.DefaultBackend $tcpservices }} {{- template "frontend-support" map $global }} {{- else if and .Global .Backends }} {{- $global := .Global }} @@ -241,7 +240,7 @@ userlist {{ $userlist.Name }} # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# # TCP SERVICES +# # TCP SERVICES (legacy, via configmap) # # # @@ -856,6 +855,7 @@ backend _error404 {{- $hosts := .p3 }} {{- $fmaps := .p4 }} {{- $defaultbackend := .p5 }} +{{- $tcpservices := .p6 }} # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @@ -915,6 +915,76 @@ frontend {{ $proxy.Name }} {{- end }} {{- end }} +{{- if $tcpservices }} + + # # # # # # # # # # # # # # # # # # # +# # +# TCP SERVICES frontends +# +{{- range $tcpport := $tcpservices }} +{{- $proxy_name := printf "_front_tcp_%d" $tcpport.Port }} +frontend {{ $proxy_name }} +{{- $tls := $tcpport.TLS }} + bind {{ $global.Bind.TCPBindIP }}:{{ $tcpport.Port }} + {{- if $tcpport.ProxyProt }} accept-proxy{{ end }} + {{- if $tls.TLSFilename }} + {{- "" }} ssl crt {{ $tls.TLSFilename }} + {{- if $tls.ALPN }} alpn {{ $tls.ALPN }}{{ end }} + {{- if $tls.CAFilename }} + {{- "" }} ca-file {{ $tls.CAFilename }} verify {{ if $tls.CAVerifyOptional}}optional{{ else }}required{{ end }} + {{- if $tls.CRLFilename }} crl-file {{ $tls.CRLFilename }}{{ end }} + {{- end }} + {{- if $tls.Ciphers }} ciphers {{ $tls.Ciphers }}{{ end }} + {{- if $tls.CipherSuites }} ciphersuites {{ $tls.CipherSuites }}{{ end }} + {{- if $tls.Options }} {{ $tls.Options }}{{ end }} + {{- end }} + mode tcp + +{{- /*------------------------------------*/}} +{{- if $global.Syslog.Endpoint }} +{{- if eq $tcpport.LogFormat "default" }} + option tcplog +{{- else if $tcpport.LogFormat }} + log-format {{ $tcpport.LogFormat }} +{{- else }} + no log +{{- end }} +{{- end }} + +{{- /*------------------------------------*/}} +{{- if $tcpport.SNIMap.HasHost }} + tcp-request inspect-delay 5s +{{- range $match := $tcpport.SNIMap.MatchFiles }} + tcp-request content set-var(req.tcpback) req.ssl_sni,lower + {{- "" }},map_{{ $match.Method }}({{ $match.Filename }}) + {{- if not $match.First }} if !{ var(req.tcpback) -m found }{{ end }} +{{- end }} +{{- end }} + +{{- /*------------------------------------*/}} +{{- range $snippet := index $global.CustomProxy $proxy_name }} + {{ $snippet }} +{{- end }} +{{- range $snippet := $tcpport.CustomConfig }} + {{ $snippet }} +{{- end }} + +{{- /*------------------------------------*/}} +{{- if $tcpport.SNIMap.HasHost }} + tcp-request content accept if { req.ssl_hello_type 1 } + use_backend %[var(req.tcpback)] if { var(req.tcpback) -m found } +{{- end }} + +{{- /*------------------------------------*/}} +{{- if $tcpport.DefaultHost }} +{{- $backend := $tcpport.DefaultHost.Backend }} +{{- if not $backend.IsEmpty }} + default_backend {{ $backend }} +{{- end }} +{{- end }} +{{- end }}{{/* range $tcpservices */}} +{{- end }}{{/* has $tcpservices */}} + {{- if $hosts.HasSSLPassthrough }} # # # # # # # # # # # # # # # # # # # @@ -958,7 +1028,9 @@ listen {{ $proxy__front__tls }} {{- /*------------------------------------*/}} use_backend %[var(req.sslpassback)] if { var(req.sslpassback) -m found } server _default_server{{ $frontend.BindName }} {{ $frontend.BindSocket }} send-proxy-v2 -{{- end }} +{{- end }}{{/* HasSSLPassthrough */}} + +{{- if $fmaps }} {{- $hasFrontingProxy := $global.Bind.HasFrontingProxy }} {{- $frontingUseProto := and $hasFrontingProxy $global.Bind.FrontingUseProto }} @@ -1253,6 +1325,7 @@ frontend {{ $proxy__front_https }} {{- end }} {{- template "defaultbackend" map $hosts $defaultbackend }} +{{- end }}{{/* has $fmaps */}} {{- end }}{{/* define "frontends" */}} {{- /*------------------------------------*/}}