Merge pull request #19073 from ramr/haproxy-cfg-mgr

Reduce HAProxy reloads - adds support to use the haproxy dynamic config api

glide.yaml

@@ -182,6 +182,11 @@ import:
 # router metrics
 - package: github.com/cockroachdb/cmux
   version: b64f5908f4945f4b11ed4a0a9d3cc1e23350866d
+# router dynamic reloads
+- package: github.com/gocarina/gocsv
+  version: a5c9099e2484f1551abb9433885e158610a25f4b
+- package: github.com/bcicen/go-haproxy
+  version: ff5824fe38bede761b873cab6e247a530e89236a
 # keep us up to date with latest DNS security fixes
 - package: github.com/miekg/dns
   version: ~v1

hack/lib/start.sh

@@ -613,6 +613,13 @@ function os::start::router() {
 	else
 		oc adm router --config="${ADMIN_KUBECONFIG}" --images="${USE_IMAGES}" --service-account=router
 	fi
+
+	# Note that when the haproxy config manager is set based on router type,
+	# the env entry may need to be always set or removed (if defaulted).
+	if [[ -n "${ROUTER_HAPROXY_CONFIG_MANAGER:-}" ]]; then
+		os::log::debug "Changing the router DC to enable the haproxy config manager"
+		oc set env dc/router -c router ROUTER_HAPROXY_CONFIG_MANAGER=true
+	fi
 }
 readonly -f os::start::router
 

images/router/haproxy/conf/haproxy-config.template

@@ -6,6 +6,7 @@
 {{- define "/var/lib/haproxy/conf/haproxy.config" }}
 {{- $workingDir := .WorkingDir }}
 {{- $defaultDestinationCA := .DefaultDestinationCA }}
+{{- $dynamicConfigManager := .DynamicConfigManager }}
 {{- $router_ip_v4_v6_mode := env "ROUTER_IP_V4_V6_MODE" "v4" }}
 
 
@@ -46,6 +47,8 @@ global
 {{- end}}
   ca-base /etc/ssl
   crt-base /etc/ssl
+  # TODO: Check if we can get reload to be faster by saving server state.
+  # server-state-file /var/lib/haproxy/run/haproxy.state
   stats socket /var/lib/haproxy/run/haproxy.sock mode 600 level admin expose-fd listeners
   stats timeout 2m
 
@@ -196,7 +199,7 @@ frontend public_ssl
   # for the SNI case, we also need to compare it in case-insensitive mode (by converting it to lowercase) as RFC 4343 says
   acl sni req.ssl_sni -m found
   acl sni_passthrough req.ssl_sni,lower,map_reg(/var/lib/haproxy/conf/os_sni_passthrough.map) -m found
-  use_backend be_tcp:%[req.ssl_sni,lower,map_reg(/var/lib/haproxy/conf/os_tcp_be.map)] if sni sni_passthrough
+  use_backend %[req.ssl_sni,lower,map_reg(/var/lib/haproxy/conf/os_tcp_be.map)] if sni sni_passthrough
 
   # if the route is SNI and NOT passthrough enter the termination flow
   use_backend be_sni if sni
@@ -367,19 +370,10 @@ backend openshift_default
 */}}
 {{- range $cfgIdx, $cfg := .State }}
   {{- if matchValues (print $cfg.TLSTermination) "" "edge" "reencrypt" }}
-    {{- if (eq $cfg.TLSTermination "") }}
 
-# Plain http backend
-backend be_http:{{$cfgIdx}}
-    {{- else if (eq $cfg.TLSTermination "edge") }}
-
-# Plain http backend but request is TLS, terminated at edge
-backend be_edge_http:{{$cfgIdx}}
-    {{ else if (eq $cfg.TLSTermination "reencrypt") }}
-
-# Secure backend which requires re-encryption
-backend be_secure:{{$cfgIdx}}
-    {{- end }}{{/* end chceck for router type */}}
+# Plain http backend or backend with TLS terminated at the edge or a
+# secure backend with re-encryption.
+backend {{genBackendNamePrefix $cfg.TLSTermination}}:{{$cfgIdx}}
   mode http
   option redispatch
   option forwardfor
@@ -473,12 +467,36 @@ backend be_secure:{{$cfgIdx}}
       {{- end }}{{/* end get serviceUnit from its name */}}
   {{- end }}{{/* end range over serviceUnitNames */}}
 
+  {{- with $dynamicConfigManager }}
+    {{- if (eq $cfg.TLSTermination "reencrypt") }}
+      {{- range $idx, $serverName := $dynamicConfigManager.GenerateDynamicServerNames $cfgIdx }}
+  server {{$serverName}} 172.4.0.4:8765 weight 0 ssl disabled check inter {{firstMatch $timeSpecPattern (index $cfg.Annotations "router.openshift.io/haproxy.health.check.interval") (env "ROUTER_BACKEND_CHECK_INTERVAL") "5000ms"}}
+        {{- if gt (len (index $cfg.Certificates (printf "%s_pod" $cfg.Host)).Contents) 0 }} verify required ca-file {{ $workingDir }}/cacerts/{{$cfgIdx}}.pem
+        {{- else }}
+          {{- if gt (len $defaultDestinationCA) 0 }} verify required ca-file {{ $defaultDestinationCA }}
+          {{- else }} verify none
+          {{- end }}
+        {{- end }}
+        {{- with $podMaxConn := index $cfg.Annotations "haproxy.router.openshift.io/pod-concurrent-connections" }}
+          {{- if (isInteger (index $cfg.Annotations "haproxy.router.openshift.io/pod-concurrent-connections")) }} maxconn {{$podMaxConn}} {{- end }}
+        {{- end}}{{/* end pod-concurrent-connections annotation */}}
+      {{- end }}{{/* end range over dynamic server names */}}
+
+    {{- else }}
+      {{- with $name := $dynamicConfigManager.ServerTemplateName $cfgIdx }}
+        {{- with $size := $dynamicConfigManager.ServerTemplateSize $cfgIdx }}
+  server-template {{$name}}- 1-{{$size}} 172.4.0.4:8765 check disabled
+        {{- end }}
+      {{- end }}
+    {{- end }}
+  {{- end }}
+
   {{- end }}{{/* end if tls==edge/none/reencrypt */}}
 
   {{- if eq $cfg.TLSTermination "passthrough" }}
 
 # Secure backend, pass through
-backend be_tcp:{{$cfgIdx}}
+backend {{genBackendNamePrefix $cfg.TLSTermination}}:{{$cfgIdx}}
 {{- if ne (env "ROUTER_SYSLOG_ADDRESS") ""}}
   option tcplog
 {{- end }}
@@ -528,6 +546,15 @@ backend be_tcp:{{$cfgIdx}}
         {{- end }}{{/* end get ServiceUnit from serviceUnitName */}}
       {{- end }}{{/* end if weight != 0 */}}
     {{- end }}{{/* end iterate over services*/}}
+
+    {{- with $dynamicConfigManager }}
+      {{- with $name := $dynamicConfigManager.ServerTemplateName $cfgIdx }}
+        {{- with $size := $dynamicConfigManager.ServerTemplateSize $cfgIdx }}
+  server-template {{$name}}- 1-{{$size}} 172.4.0.4:8765 check disabled
+        {{- end }}
+      {{- end }}
+    {{- end }}
+
   {{- end }}{{/*end tls==passthrough*/}}
 
 {{- end }}{{/* end loop over routes */}}
@@ -564,6 +591,7 @@ backend be_tcp:{{$cfgIdx}}
 {{ end -}}{{/* end http host map template */}}
 
 
+
 {{/*
     os_edge_reencrypt_be.map : contains a mapping of www.example.com -> <service name>. This map is similar to os_http_be.map but for tls routes.
                          by attaching prefix: be_edge_http for edge terminated routes
@@ -590,7 +618,7 @@ backend be_tcp:{{$cfgIdx}}
 
 {{/*
     os_tcp_be.map: contains a mapping of www.example.com -> <service name>.  This map is used to discover the correct backend
-                        by attaching a prefix (be_tcp: or be_secure:) by use_backend statements if acls are matched.
+                        by use_backend statements if acls are matched.
 */}}
 {{ define "/var/lib/haproxy/conf/os_tcp_be.map" -}}
 {{     range $idx, $line := generateHAProxyMap . -}}