*: fix drainer can't send request correctly to pump when set compress…

…or=gzip (#1186) (#1198) close #1152
pingcap · Aug 18, 2022 · 2e8bd50 · 2e8bd50
1 parent 1012ec7
commit 2e8bd50
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 6 deletions.
diff --git a/drainer/util.go b/drainer/util.go
@@ -43,7 +43,9 @@ import (
 
 const (
 	maxKafkaMsgSize = 1 << 30
-	maxGrpcMsgSize  = int(^uint(0) >> 1)
+	// max grpc message size, leave 4MB as buffer. Because when grpc decompresses messages, it will leave a few buffer
+	// for this, which overflows the int64: https://github.com/grpc/grpc-go/blob/v1.44.0/rpc_util.go#L742
+	maxGrpcMsgSize = int(^uint(0)>>1) - 4*1024*1024
 )
 
 // taskGroup is a wrapper of `sync.WaitGroup`.

diff --git a/pump/config.go b/pump/config.go
@@ -32,10 +32,12 @@ import (
 )
 
 const (
-	defaultEtcdDialTimeout   = 5 * time.Second
-	defaultEtcdURLs          = "http://127.0.0.1:2379"
-	defaultListenAddr        = "127.0.0.1:8250"
-	defaultMaxMsgSize        = int(^uint(0) >> 1) // max grpc message size
+	defaultEtcdDialTimeout = 5 * time.Second
+	defaultEtcdURLs        = "http://127.0.0.1:2379"
+	defaultListenAddr      = "127.0.0.1:8250"
+	// max grpc message size, leave 4MB as buffer. Because when grpc decompresses messages, it will leave a few buffer
+	// for this, which overflows the int64: https://github.com/grpc/grpc-go/blob/v1.44.0/rpc_util.go#L742
+	defaultMaxMsgSize        = int(^uint(0)>>1) - 4*1024*1024
 	defaultHeartbeatInterval = 2
 	defaultGC                = "7"
 	defaultDataDir           = "data.pump"

diff --git a/tests/cache_table/run.sh b/tests/cache_table/run.sh
@@ -4,7 +4,7 @@ set -e
 
 cd "$(dirname "$0")"
 
-run_drainer &
+run_drainer --compressor gzip &
 
 sleep 3
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,7 +4,7 @@ set -e @@
     cd "$(dirname "$0")"
-    run_drainer &
+    run_drainer --compressor gzip &
     sleep 3
@@ Expand Down @@