slack-vitess-r15.0.5: forward-port consul topo limits PR #111 (#297)

* `slack-vitess-r14.0.5`: allow conn overrides in consul topo (#111) * `slack-vitess-r14.0.5`: allow conn overrides in consul topo Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com> * fix e2e test Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com> --------- Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com> * Update flags tests that didn't exist in v14 Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com> --------- Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com>
slackhq · Jul 9, 2024 · 2dfc4d3 · 2dfc4d3
1 parent 462f380
commit 2dfc4d3
Show file tree

Hide file tree

Showing 7 changed files with 23 additions and 1 deletion.
diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt
@@ -213,9 +213,12 @@ Flags:
       --tablet_manager_grpc_key string                              the key to use to connect
       --tablet_manager_grpc_server_name string                      the server name to use to validate server certificate
       --tablet_manager_protocol string                              Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc")
+      --topo_consul_idle_conn_timeout duration                      Maximum amount of time to pool idle connections. (default 1m30s)
       --topo_consul_lock_delay duration                             LockDelay for consul session. (default 15s)
       --topo_consul_lock_session_checks string                      List of checks for consul session. (default "serfHealth")
       --topo_consul_lock_session_ttl string                         TTL for consul session.
+      --topo_consul_max_conns_per_host int                          Maximum number of consul connections per host.
+      --topo_consul_max_idle_conns int                              Maximum number of idle consul connections. (default 100)
       --topo_consul_watch_poll_duration duration                    time of the long poll for watch queries. (default 30s)
       --topo_etcd_lease_ttl int                                     Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30)
       --topo_etcd_tls_ca string                                     path to the ca to use to validate the server cert when connecting to the etcd topo server

diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt
@@ -151,9 +151,12 @@ Flags:
       --tablet_refresh_interval duration                                 Tablet refresh interval. (default 1m0s)
       --tablet_refresh_known_tablets                                     Whether to reload the tablet's address/port map from topo in case they change. (default true)
       --tablet_url_template string                                       Format string describing debug tablet url formatting. See getTabletDebugURL() for how to customize this. (default "http://{{ "{{.GetTabletHostPort}}" }}")
+      --topo_consul_idle_conn_timeout duration                           Maximum amount of time to pool idle connections. (default 1m30s)
       --topo_consul_lock_delay duration                                  LockDelay for consul session. (default 15s)
       --topo_consul_lock_session_checks string                           List of checks for consul session. (default "serfHealth")
       --topo_consul_lock_session_ttl string                              TTL for consul session.
+      --topo_consul_max_conns_per_host int                               Maximum number of consul connections per host.
+      --topo_consul_max_idle_conns int                                   Maximum number of idle consul connections. (default 100)
       --topo_consul_watch_poll_duration duration                         time of the long poll for watch queries. (default 30s)
       --topo_etcd_lease_ttl int                                          Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30)
       --topo_etcd_tls_ca string                                          path to the ca to use to validate the server cert when connecting to the etcd topo server

diff --git a/go/flags/endtoend/vtgate.txt b/go/flags/endtoend/vtgate.txt
@@ -204,9 +204,12 @@ Flags:
       --tablet_refresh_known_tablets                                     Whether to reload the tablet's address/port map from topo in case they change. (default true)
       --tablet_types_to_wait strings                                     Wait till connected for specified tablet types during Gateway initialization. Should be provided as a comma-separated set of tablet types.
       --tablet_url_template string                                       Format string describing debug tablet url formatting. See getTabletDebugURL() for how to customize this. (default "http://{{ "{{.GetTabletHostPort}}" }}")
+      --topo_consul_idle_conn_timeout duration                           Maximum amount of time to pool idle connections. (default 1m30s)
       --topo_consul_lock_delay duration                                  LockDelay for consul session. (default 15s)
       --topo_consul_lock_session_checks string                           List of checks for consul session. (default "serfHealth")
       --topo_consul_lock_session_ttl string                              TTL for consul session.
+      --topo_consul_max_conns_per_host int                               Maximum number of consul connections per host.
+      --topo_consul_max_idle_conns int                                   Maximum number of idle consul connections. (default 100)
       --topo_consul_watch_poll_duration duration                         time of the long poll for watch queries. (default 30s)
       --topo_etcd_lease_ttl int                                          Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30)
       --topo_etcd_tls_ca string                                          path to the ca to use to validate the server cert when connecting to the etcd topo server

diff --git a/go/flags/endtoend/vtorc.txt b/go/flags/endtoend/vtorc.txt
@@ -89,9 +89,12 @@ Flags:
       --tablet_manager_protocol string                              Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc")
       --tolerable-replication-lag duration                          Amount of replication lag that is considered acceptable for a tablet to be eligible for promotion when Vitess makes the choice of a new primary in PRS
       --topo-information-refresh-duration duration                  Timer duration on which VTOrc refreshes the keyspace and vttablet records from the topology server (default 15s)
+      --topo_consul_idle_conn_timeout duration                      Maximum amount of time to pool idle connections. (default 1m30s)
       --topo_consul_lock_delay duration                             LockDelay for consul session. (default 15s)
       --topo_consul_lock_session_checks string                      List of checks for consul session. (default "serfHealth")
       --topo_consul_lock_session_ttl string                         TTL for consul session.
+      --topo_consul_max_conns_per_host int                          Maximum number of consul connections per host.
+      --topo_consul_max_idle_conns int                              Maximum number of idle consul connections. (default 100)
       --topo_consul_watch_poll_duration duration                    time of the long poll for watch queries. (default 30s)
       --topo_etcd_lease_ttl int                                     Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30)
       --topo_etcd_tls_ca string                                     path to the ca to use to validate the server cert when connecting to the etcd topo server

diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt
@@ -355,9 +355,12 @@ Flags:
       --tablet_manager_protocol string                                   Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc")
       --tablet_protocol string                                           Protocol to use to make queryservice RPCs to vttablets. (default "grpc")
       --throttle_tablet_types string                                     Comma separated VTTablet types to be considered by the throttler. default: 'replica'. example: 'replica,rdonly'. 'replica' always implicitly included (default "replica")
+      --topo_consul_idle_conn_timeout duration                           Maximum amount of time to pool idle connections. (default 1m30s)
       --topo_consul_lock_delay duration                                  LockDelay for consul session. (default 15s)
       --topo_consul_lock_session_checks string                           List of checks for consul session. (default "serfHealth")
       --topo_consul_lock_session_ttl string                              TTL for consul session.
+      --topo_consul_max_conns_per_host int                               Maximum number of consul connections per host.
+      --topo_consul_max_idle_conns int                                   Maximum number of idle consul connections. (default 100)
       --topo_consul_watch_poll_duration duration                         time of the long poll for watch queries. (default 30s)
       --topo_etcd_lease_ttl int                                          Lease TTL for locks and leader election. The client will use KeepAlive to keep the lease going. (default 30)
       --topo_etcd_tls_ca string                                          path to the ca to use to validate the server cert when connecting to the etcd topo server

diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt
@@ -127,9 +127,12 @@ Flags:
       --tablet_manager_grpc_server_name string                           the server name to use to validate server certificate
       --tablet_manager_protocol string                                   Protocol to use to make tabletmanager RPCs to vttablets. (default "grpc")
       --tablet_refresh_interval duration                                 Interval at which vtgate refreshes tablet information from topology server. (default 10s)
+      --topo_consul_idle_conn_timeout duration                           Maximum amount of time to pool idle connections. (default 1m30s)
       --topo_consul_lock_delay duration                                  LockDelay for consul session. (default 15s)
       --topo_consul_lock_session_checks string                           List of checks for consul session. (default "serfHealth")
       --topo_consul_lock_session_ttl string                              TTL for consul session.
+      --topo_consul_max_conns_per_host int                               Maximum number of consul connections per host.
+      --topo_consul_max_idle_conns int                                   Maximum number of idle consul connections. (default 100)
       --topo_consul_watch_poll_duration duration                         time of the long poll for watch queries. (default 30s)
       --topo_zk_auth_file string                                         auth to use when connecting to the zk topo server, file contents should be <scheme>:<auth>, e.g., digest:user:pass
       --topo_zk_base_timeout duration                                    zk base timeout (see zk.Connect) (default 30s)

diff --git a/go/vt/topo/consultopo/server.go b/go/vt/topo/consultopo/server.go
@@ -37,6 +37,7 @@ import (
 
 var (
 	consulAuthClientStaticFile string
+	consulConfig               = api.DefaultConfig()
 	// serfHealth is the default check from consul
 	consulLockSessionChecks = "serfHealth"
 	consulLockSessionTTL    string
@@ -52,6 +53,9 @@ func registerServerFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&consulLockSessionChecks, "topo_consul_lock_session_checks", consulLockSessionChecks, "List of checks for consul session.")
 	fs.StringVar(&consulLockSessionTTL, "topo_consul_lock_session_ttl", consulLockSessionTTL, "TTL for consul session.")
 	fs.DurationVar(&consulLockDelay, "topo_consul_lock_delay", consulLockDelay, "LockDelay for consul session.")
+	fs.IntVar(&consulConfig.Transport.MaxConnsPerHost, "topo_consul_max_conns_per_host", consulConfig.Transport.MaxConnsPerHost, "Maximum number of consul connections per host.")
+	fs.IntVar(&consulConfig.Transport.MaxIdleConns, "topo_consul_max_idle_conns", consulConfig.Transport.MaxIdleConns, "Maximum number of idle consul connections.")
+	fs.DurationVar(&consulConfig.Transport.IdleConnTimeout, "topo_consul_idle_conn_timeout", consulConfig.Transport.IdleConnTimeout, "Maximum amount of time to pool idle connections.")
 }
 
 // ClientAuthCred credential to use for consul clusters
@@ -130,7 +134,7 @@ func NewServer(cell, serverAddr, root string) (*Server, error) {
 	if err != nil {
 		return nil, err
 	}
-	cfg := api.DefaultConfig()
+	cfg := consulConfig
 	cfg.Address = serverAddr
 	if creds != nil {
 		if creds[cell] != nil {