[HAProxy] Testing.

aptos-labs · Jul 10, 2024 · cc437cd · cc437cd
1 parent 68f047c
commit cc437cd
Show file tree

Hide file tree

Showing 5 changed files with 105 additions and 265 deletions.
diff --git a/docker/compose/aptos-node/haproxy.cfg b/docker/compose/aptos-node/haproxy.cfg
@@ -1,139 +1,63 @@
+# Config manual: https://www.haproxy.com/documentation/haproxy-configuration-manual/latest/
+
+## Global settings
 global
+    # Specify the stdout log format and size
     log stdout len 10240 format raw local0
 
-    # Config manual: https://cbonte.github.io/haproxy-dconv/2.5/configuration.html
-    # magic values : terraform/helm/aptos-node/values.yaml
-
-    # 256 connections/sec * upgrade_to(30 sec)
-    maxconn 8192
-    # This limits the whole HA Proxy impacting both validators and other frontends
-    # maxconnrate 128
-    nbthread 4
+    # Limit the maximum number of connections to 1000 (this is ~10x the validator set size)
+    maxconn 1000
 
-    #4MB for client facing sndbuf/rcvbuf. -- 100Mb/s with 300 mili latency (e.g., us-asia)
-    tune.sndbuf.client 4194304	#tcpBufSize
-    tune.rcvbuf.client 4194304	#tcpBufSize
+    # Limit the maximum number of connections per second to 500
+    maxconnrate 500
 
+    # Limit user privileges
     user nobody
 
-## TCP port defaults
+## Default settings
 defaults
+    # Enable logging of events and traffic
     log global
+
+    # Set the default mode to TCP
     mode tcp
-    #option tcplog
+
+    # Don't log normal events
     option dontlog-normal
-    log-format "%ci:%cp - %sp[%rt] [%t] %ft %Tw/%Tc/%Tt %B [%ts] %ac/%fc/%bc/%sc/%rc %sq/%bq"
-    maxconn 8192		#Validator network mesh + FN x2
-    retries 3
-    timeout queue 5s  #limits num of concurrent connections. Not clear if t/o connect is needed. #https://www.papertrail.com/solution/tips/haproxy-logging-how-to-tune-timeouts-for-performance/
-    timeout connect 5s
-    # enough for 1 successfull + 5 unsuccessfull HB(10 sec interval) + 20 sec timeout
-    timeout server 80s
-    timeout client 80s
-
-    timeout client-fin 3s #How long to hold an interrupted client connection.
-    timeout server-fin 1s
 
+    # Set timeouts for new connections
+    timeout queue 10s
+    timeout connect 10s
+
+    # Prevent long-running HTTP requests
+    timeout http-request 60s
+    timeout http-keep-alive 2s
+
+## Specify the validator frontend
 frontend fe-validator
     bind :6180
     default_backend validator
 
     # Deny requests from blocked IPs
     tcp-request connection silent-drop if { src -n -f /usr/local/etc/haproxy/blocked.ips }
 
-    # We deem a connection rate high when an IP is attempting to reconnect more than twice a min
-    acl ip_high_conn_rate sc0_conn_rate gt 12
-
-    stick-table type ip size 128K expire 30m store gpc1,conn_rate(1m),bytes_out_cnt	##about 500MB of memory
-    tcp-request connection track-sc0 src 						   #update table with src ip as key, store in sc0
-
-    #We Count rate-limit manualy -- Will be more CPU intensieve but will allow whitelists to enter and up to rateLimitSession non blacklisted IPs.
-    tcp-request connection track-sc1 int(1) table CONN_RATE
-
-    #This connection is silently dropped no reason to count it for rateLimitSession
-    tcp-request connection sc-inc-gpc1(1) unless ip_high_conn_rate
-
-    # an IP is rejected due to to many unsucsessfull tcp attempts
-    #-1- Enforce connection rate limit
-    tcp-request connection silent-drop if ip_high_conn_rate
-
-    #an IP that had a sucessfull connection.
-    #-2- Allow Whitelist
-    tcp-request connection accept if { sc0_get_gpc1() ge 1 }
-
-    #-3- Enforce RateLimit. Connection attempts by *new* IPs/sec
-    tcp-request connection reject if { sc1_gpc1_rate(CONN_RATE) gt  256 } #rateLimitSession
-
-    # This is a successfull connection i.e., was sent more than 16K bytes in the last 30 min
-    #tcp-request session sc-set-gpt0(0) int(...)  if { sc0_kbytes_out gt 16 }
-    #<2> Mark Whitelist
-    tcp-request session sc-inc-gpc1(0) if { sc0_kbytes_out gt 4 }
-
-
+## Specify the validator backend
 backend validator
-    default-server maxconn 8192
     server validator validator:6180
 
+## Specify the VFN frontend
 frontend fe-fullnode
     bind :6181
     default_backend validator-fn
 
     # Deny requests from blocked IPs
     tcp-request connection silent-drop if { src -n -f /usr/local/etc/haproxy/blocked.ips }
 
-    acl ip_high_conn_rate sc0_conn_rate gt 12
-
-    stick-table type ip size 128K expire 30m store gpc0,gpc1,conn_rate(1m),bytes_out_cnt	##about 500MB of memory
-    tcp-request connection track-sc0 src 						   #update table with src ip as key, store in sc0
-
-    #We Count rate-limit manualy -- Will be more CPU intensieve but will allow whitelists to enter and up to rateLimitSession non blacklisted IPs.
-    tcp-request connection track-sc1 int(1) table CONN_RATE
-
-    #This connection is silently dropped no reason to count it for rateLimitSession
-    tcp-request connection sc-inc-gpc1(1) unless ip_high_conn_rate
-
-    # an IP is rejected due to to many unsucsessfull tcp attempts
-    #-1- Enforce connection rate limit
-    tcp-request connection silent-drop if ip_high_conn_rate
-
-    #an IP that had a sucessfull connection.
-    #-2- Allow Whitelist
-    tcp-request connection accept if { sc0_get_gpc1() ge 1 }
-
-    #-3- Enforce RateLimit. Connection attempts by *new* IPs/sec
-    tcp-request connection reject if { sc1_gpc1_rate(CONN_RATE) gt  256 } #rateLimitSession
-
-    # This is a successfull connection i.e., was sent more than 16K bytes in the last 30 min
-    #tcp-request session sc-set-gpt0(0) int(...)  if { sc0_kbytes_out gt 16 }
-    #<2> Mark Whitelist
-    tcp-request session sc-inc-gpc1(0) if { sc0_kbytes_out gt 4 }
-
-
+## Specify the VFN backend
 backend validator-fn
-    default-server maxconn 16
     server validator validator:6181
 
-#CONNRATE holds only entry with key 1: used for determening global conn rate
-backend CONN_RATE
-    stick-table type integer size 1 expire 10m store gpc1,gpc1_rate(1s)
-
-##################  HTTP: metrics & API
-defaults
-	mode http
-        retries 3
-        timeout queue 5s  #limits num of concurrent connections. Not clear if t/o connect is needed. #https://www.papertrail.com/solution/tips/haproxy-logging-how-to-tune-timeouts-for-performance/
-        timeout connect 5s
-        timeout server 60s #what makes sense? for silence between nodes?
-        timeout client 60s
-
-        timeout client-fin 3s #How long to hold an interrupted client connection.
-        timeout server-fin 1s
-
-	timeout http-request 60s #len of http request
-	timeout http-keep-alive 2s
-
-        rate-limit sessions 256
-
+## Specify the validator metrics frontend
 frontend validator-metrics
     mode http
     option httplog
@@ -142,13 +66,16 @@ frontend validator-metrics
 
     # Deny requests from blocked IPs
     tcp-request connection reject if { src -n -f /usr/local/etc/haproxy/blocked.ips }
+
+    ## Add the forwarded header
     http-request add-header Forwarded "for=%ci"
 
+## Specify the validator metrics backend
 backend validator-metrics
     mode http
-    default-server maxconn 8
     server validator validator:9101
 
+## Specify the validator API frontend
 frontend validator-api
     mode http
     option httplog
@@ -157,17 +84,19 @@ frontend validator-api
 
     # Deny requests from blocked IPs
     tcp-request connection reject if { src -n -f /usr/local/etc/haproxy/blocked.ips }
+
+    ## Add the forwarded header
     http-request add-header Forwarded "for=%ci"
 
+## Specify the validator API backend
 backend validator-api
     mode http
-    default-server maxconn 128
     server validator validator:8080
 
+## Specify the metrics frontend
 frontend stats
     mode http
-    bind :9102
-    option http-use-htx
+    bind :9101
     http-request use-service prometheus-exporter if { path /metrics }
     stats enable
     stats uri /stats