Make AM grpc MaxRecvMsgSize and MaxSendMsgSize configurable

Signed-off-by: Emmanuel Lodovice <lodovice@amazon.com>

CHANGELOG.md

@@ -6,6 +6,7 @@
 * [CHANGE] Ingester: Creating label `native-histogram-sample` on the `cortex_discarded_samples_total` to keep track of discarded native histogram samples. #5289
 * [ENHANCEMENT] Distributor/Ingester: Add span on push path #5319
 * [FEATURE] Store Gateway: Add `max_downloaded_bytes_per_request` to limit max bytes to download per store gateway request.
+* [FEATURE] Added 2 flags `-alertmanager.alertmanager-client.grpc-max-send-msg-size` and ` -alertmanager.alertmanager-client.grpc-max-recv-msg-size` to configure alert manager grpc client message size limits. #5338
 * [ENHANCEMENT] Support object storage backends for runtime configuration file. #5292
 * [ENHANCEMENT] Query Frontend: Reject subquery with too small step size. #5323
 * [ENHANCEMENT] Compactor: Exposing Thanos accept-malformed-index to Cortex compactor. #5334

docs/configuration/config-file-reference.md

@@ -446,6 +446,14 @@ alertmanager_client:
   # CLI flag: -alertmanager.alertmanager-client.grpc-compression
   [grpc_compression: <string> | default = ""]
 
+  # gRPC client max receive message size (bytes).
+  # CLI flag: -alertmanager.alertmanager-client.grpc-max-recv-msg-size
+  [max_recv_msg_size: <int> | default = 16777216]
+
+  # gRPC client max send message size (bytes).
+  # CLI flag: -alertmanager.alertmanager-client.grpc-max-send-msg-size
+  [max_send_msg_size: <int> | default = 4194304]
+
 # The interval between persisting the current alertmanager state (notification
 # log and silences) to object storage. This is only used when sharding is
 # enabled. This state is read when all replicas for a shard can not be

pkg/alertmanager/alertmanager_client.go

@@ -38,6 +38,8 @@ type ClientConfig struct {
 	TLSEnabled      bool             `yaml:"tls_enabled"`
 	TLS             tls.ClientConfig `yaml:",inline"`
 	GRPCCompression string           `yaml:"grpc_compression"`
+	MaxRecvMsgSize  int              `yaml:"max_recv_msg_size"`
+	MaxSendMsgSize  int              `yaml:"max_send_msg_size"`
 }
 
 // RegisterFlagsWithPrefix registers flags with prefix.
@@ -46,6 +48,8 @@ func (cfg *ClientConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet)
 	f.DurationVar(&cfg.RemoteTimeout, prefix+".remote-timeout", 2*time.Second, "Timeout for downstream alertmanagers.")
 	f.StringVar(&cfg.GRPCCompression, prefix+".grpc-compression", "", "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)")
 	cfg.TLS.RegisterFlagsWithPrefix(prefix, f)
+	f.IntVar(&cfg.MaxRecvMsgSize, prefix+".grpc-max-recv-msg-size", 16*1024*1024, "gRPC client max receive message size (bytes).")
+	f.IntVar(&cfg.MaxSendMsgSize, prefix+".grpc-max-send-msg-size", 4*1024*1024, "gRPC client max send message size (bytes).")
 }
 
 type alertmanagerClientsPool struct {
@@ -55,8 +59,8 @@ type alertmanagerClientsPool struct {
 func newAlertmanagerClientsPool(discovery client.PoolServiceDiscovery, amClientCfg ClientConfig, logger log.Logger, reg prometheus.Registerer) ClientsPool {
 	// We prefer sane defaults instead of exposing further config options.
 	grpcCfg := grpcclient.Config{
-		MaxRecvMsgSize:      16 * 1024 * 1024,
-		MaxSendMsgSize:      4 * 1024 * 1024,
+		MaxRecvMsgSize:      amClientCfg.MaxRecvMsgSize,
+		MaxSendMsgSize:      amClientCfg.MaxSendMsgSize,
 		GRPCCompression:     amClientCfg.GRPCCompression,
 		RateLimit:           0,
 		RateLimitBurst:      0,