=== RUN TestStoreRangeMergeRHSLeaseExpiration I200415 01:08:28.760373 67336 gossip/gossip.go:404 [n1] NodeDescriptor set to node_id:1 address: attrs:<> locality:<> ServerVersion: build_tag:"" started_at:0 cluster_name:"" sql_address: W200415 01:08:28.840941 67336 gossip/gossip.go:1524 [n2] no incoming or outgoing connections I200415 01:08:28.841939 67336 gossip/gossip.go:404 [n2] NodeDescriptor set to node_id:2 address: attrs:<> locality:<> ServerVersion: build_tag:"" started_at:0 cluster_name:"" sql_address: I200415 01:08:28.847070 67262 gossip/client.go:124 [n2] started gossip client to 127.0.0.1:35413 I200415 01:08:28.923574 67336 kv/kvserver/replica_command.go:397 [s1,r1/1:/M{in-ax}] initiating a split of this range at key "b" [r2] (manual) I200415 01:08:28.984473 67336 kv/kvserver/replica_command.go:1602 [s1,r1/1:{/Min-b}] change replicas (add [(n2,s2):2LEARNER] remove []): existing descriptor r1:{/Min-b} [(n1,s1):1, next=2, gen=1] I200415 01:08:29.016743 67336 kv/kvserver/replica_raft.go:247 [s1,r1/1:{/Min-b},txn=b4f7b1db] proposing SIMPLE(l2) ADD_REPLICA[(n2,s2):2LEARNER]: after=[(n1,s1):1 (n2,s2):2LEARNER] next=3 I200415 01:08:29.036374 67336 kv/kvserver/store_snapshot.go:932 [s1,r1/1:{/Min-b}] sending LEARNER snapshot 3267f83d at applied index 22 I200415 01:08:29.039777 67336 kv/kvserver/store_snapshot.go:975 [s1,r1/1:{/Min-b}] streamed snapshot to (n2,s2):2LEARNER: kv pairs: 21, log entries: 0, rate-limit: 8.0 MiB/sec, 0.01s I200415 01:08:29.045082 67677 kv/kvserver/replica_raftstorage.go:780 [s2,r1/2:{-}] applying LEARNER snapshot [id=3267f83d index=22] I200415 01:08:29.052524 67677 kv/kvserver/replica_raftstorage.go:801 [s2,r1/2:{/Min-b}] applied LEARNER snapshot [total=6ms ingestion=4@3ms id=3267f83d index=22] I200415 01:08:29.066665 67336 kv/kvserver/replica_command.go:1602 [s1,r1/1:{/Min-b}] change replicas (add [(n2,s2):2] remove []): existing descriptor r1:{/Min-b} [(n1,s1):1, (n2,s2):2LEARNER, next=3, gen=2] I200415 01:08:29.080401 67336 kv/kvserver/replica_raft.go:247 [s1,r1/1:{/Min-b},txn=40e0f4c9] proposing SIMPLE(v2) ADD_REPLICA[(n2,s2):2]: after=[(n1,s1):1 (n2,s2):2] next=3 I200415 01:08:29.132389 67336 kv/kvserver/replica_command.go:1602 [s1,r2/1:{b-/Max}] change replicas (add [(n2,s2):2LEARNER] remove []): existing descriptor r2:{b-/Max} [(n1,s1):1, next=2, gen=1] I200415 01:08:29.168269 67336 kv/kvserver/replica_raft.go:247 [s1,r2/1:{b-/Max},txn=6979c1a4] proposing SIMPLE(l2) ADD_REPLICA[(n2,s2):2LEARNER]: after=[(n1,s1):1 (n2,s2):2LEARNER] next=3 I200415 01:08:29.191298 67336 kv/kvserver/store_snapshot.go:932 [s1,r2/1:{b-/Max}] sending LEARNER snapshot 3cdff901 at applied index 16 I200415 01:08:29.194039 67336 kv/kvserver/store_snapshot.go:975 [s1,r2/1:{b-/Max}] streamed snapshot to (n2,s2):2LEARNER: kv pairs: 68, log entries: 0, rate-limit: 8.0 MiB/sec, 0.01s I200415 01:08:29.207335 67787 kv/kvserver/raft_snapshot_queue.go:126 [raftsnapshot,s1,r2/1:{b-/Max}] skipping snapshot; replica is likely a learner in the process of being added: (n2,s2):2LEARNER I200415 01:08:29.231552 67816 kv/kvserver/replica_raftstorage.go:780 [s2,r2/2:{-}] applying LEARNER snapshot [id=3cdff901 index=16] I200415 01:08:29.250101 67816 kv/kvserver/replica_raftstorage.go:801 [s2,r2/2:{b-/Max}] applied LEARNER snapshot [total=18ms ingestion=4@10ms id=3cdff901 index=16] I200415 01:08:29.306782 67336 kv/kvserver/replica_command.go:1602 [s1,r2/1:{b-/Max}] change replicas (add [(n2,s2):2] remove []): existing descriptor r2:{b-/Max} [(n1,s1):1, (n2,s2):2LEARNER, next=3, gen=2] I200415 01:08:29.419564 67336 kv/kvserver/replica_raft.go:247 [s1,r2/1:{b-/Max},txn=196274ea] proposing SIMPLE(v2) ADD_REPLICA[(n2,s2):2]: after=[(n1,s1):1 (n2,s2):2] next=3 I200415 01:08:29.550449 67779 kv/kvserver/replica_command.go:663 [s1,r1/1:{/Min-b}] initiating a merge of r2:{b-/Max} [(n1,s1):1, (n2,s2):2, next=3, gen=3] into this range (manual) I200415 01:08:29.666560 67336 kv/kvserver/client_test.go:1420 test clock advanced to: 1.800000125,0 I200415 01:08:30.387572 67313 gossip/gossip.go:1538 [n1] node has connected to cluster via gossip I200415 01:08:30.405309 68294 kv/kvserver/client_merge_test.go:1464 starting req 0 I200415 01:08:30.406657 68242 kv/kvserver/client_merge_test.go:1464 starting req 1 I200415 01:08:30.407816 68228 kv/kvserver/client_merge_test.go:1464 starting req 2 I200415 01:08:30.409495 68256 kv/kvserver/client_merge_test.go:1464 starting req 3 I200415 01:08:30.410749 68257 kv/kvserver/client_merge_test.go:1464 starting req 4 I200415 01:08:30.413619 68295 kv/kvserver/client_merge_test.go:1464 starting req 5 I200415 01:08:30.413914 68326 kv/kvserver/client_merge_test.go:1464 starting req 6 I200415 01:08:30.417413 68164 kv/kvserver/client_merge_test.go:1464 starting req 7 I200415 01:08:30.418535 68341 kv/kvserver/client_merge_test.go:1464 starting req 8 I200415 01:08:30.420748 68327 kv/kvserver/client_merge_test.go:1464 starting req 9 I200415 01:08:30.859535 67590 gossip/gossip.go:1538 [n2] node has connected to cluster via gossip W200415 01:09:17.966985 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:09:29.610826 67885 kv/kvserver/txnwait/queue.go:517 [s1,r1/1:{/Min-b}] pusher 00000000: have been waiting 60.00s for pushee a87af617 W200415 01:09:32.361485 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:10:09.580206 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:10:42.028784 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:11:15.047289 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:11:33.900210 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:12:19.112565 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:12:40.402770 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:13:08.989961 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:13:44.136073 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:14:19.936848 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:14:39.156050 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:15:26.025984 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:15:41.315677 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:16:31.782511 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:16:55.318703 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:17:19.357951 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:18:06.151719 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:18:15.652391 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:18:54.401256 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:19:20.805572 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:19:52.750378 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:20:34.783641 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:20:39.831617 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:21:24.383363 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:21:52.033193 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:22:09.518967 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:23:05.412022 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:23:11.319205 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:23:54.699440 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:23:59.970637 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:25:06.272590 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:25:12.534484 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:26:06.316258 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:26:07.745584 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:27:05.894769 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:27:20.917249 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:28:13.179711 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:28:15.822295 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:28:28.967432 511956 kv/kvserver/intentresolver/intent_resolver.go:441 failed to push during intent resolution: failed to push meta={id=a87af617 key=/Local/Range/Min/RangeDescriptor pri=0.02333550 epo=0 ts=0.000000123,323 min=0.000000123,323 seq=0} lock=true stat=PENDING rts=0,0 wto=false max=0,0 W200415 01:29:11.321364 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:29:19.922279 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store E200415 01:29:28.881288 512053 kv/kvserver/consistency_queue.go:135 [consistencyChecker,s2,r2/2:{b-/Max}] aborted in distSender: context deadline exceeded E200415 01:29:28.882244 512053 kv/kvserver/queue.go:1089 [consistencyChecker,s2,r2/2:{b-/Max}] operation "consistencyChecker queue process replica 2" timed out after 1m0s: aborted in distSender: context deadline exceeded W200415 01:30:07.842229 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:30:16.543385 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:31:05.418002 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:31:09.092223 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:32:05.413426 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:32:07.486751 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:33:04.628678 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:33:16.298423 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:34:09.603055 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:34:11.527289 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:35:01.485946 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:35:05.030524 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:36:03.299356 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:36:06.309053 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:37:03.188464 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:37:20.857725 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:38:04.055571 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:38:31.683806 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:39:00.798147 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:39:18.594529 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store E200415 01:39:28.913159 739335 kv/kvserver/consistency_queue.go:135 [consistencyChecker,s2,r2/2:{b-/Max}] aborted in distSender: context deadline exceeded E200415 01:39:28.913914 739335 kv/kvserver/queue.go:1089 [consistencyChecker,s2,r2/2:{b-/Max}] operation "consistencyChecker queue process replica 2" timed out after 1m0s: aborted in distSender: context deadline exceeded W200415 01:40:03.074577 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:40:06.386573 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:40:57.347571 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:41:16.976189 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:41:50.987229 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:42:04.082268 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:42:56.753558 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:42:57.964724 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:43:53.654966 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:43:57.677770 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:44:44.482433 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:44:49.272634 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:45:39.797523 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:45:49.781284 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:46:38.436464 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:46:58.011092 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:47:30.364512 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:48:11.982302 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:48:26.293786 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:49:01.063842 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:49:16.725848 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store E200415 01:49:28.948839 922078 kv/kvserver/consistency_queue.go:135 [consistencyChecker,s2,r2/2:{b-/Max}] aborted in distSender: context deadline exceeded E200415 01:49:28.949573 922078 kv/kvserver/queue.go:1089 [consistencyChecker,s2,r2/2:{b-/Max}] operation "consistencyChecker queue process replica 2" timed out after 1m0s: aborted in distSender: context deadline exceeded W200415 01:50:10.525072 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:50:21.837774 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store W200415 01:50:58.758062 67574 kv/kvserver/store_rebalancer.go:223 [s2,store-rebalancer] StorePool missing descriptor for local store W200415 01:51:19.137187 67436 kv/kvserver/store_rebalancer.go:223 [s1,store-rebalancer] StorePool missing descriptor for local store panic: test timed out after 45m0s goroutine 972034 [running]: testing.(*M).startAlarm.func1() /usr/local/go/src/testing/testing.go:1377 +0x11c created by time.goFunc /usr/local/go/src/time/sleep.go:168 +0x52 goroutine 1 [chan receive, 42 minutes]: testing.(*T).Run(0xc0004f8200, 0x6171bd8, 0x25, 0x62a3870, 0x1) /usr/local/go/src/testing/testing.go:961 +0x68a testing.runTests.func1(0xc0004f8200) /usr/local/go/src/testing/testing.go:1202 +0xa7 testing.tRunner(0xc0004f8200, 0xc00075fab0) /usr/local/go/src/testing/testing.go:909 +0x19a testing.runTests(0xc0002de440, 0x99983e0, 0x260, 0x260, 0x0) /usr/local/go/src/testing/testing.go:1200 +0x522 testing.(*M).Run(0xc00015e100, 0x0) /usr/local/go/src/testing/testing.go:1117 +0x300 github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestMain(0xc00015e100) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/main_test.go:53 +0x298 main.main() _testmain.go:1266 +0x224 goroutine 50 [syscall, 45 minutes]: os/signal.signal_recv(0xc2ea01) /usr/local/go/src/runtime/sigqueue.go:147 +0x9c os/signal.loop() /usr/local/go/src/os/signal/signal_unix.go:23 +0x30 created by os/signal.init.0 /usr/local/go/src/os/signal/signal_unix.go:29 +0x4f goroutine 36 [chan receive]: github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon() /go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:73 +0x96 created by github.com/cockroachdb/cockroach/pkg/util/log.init.7 /go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:45 +0x43 goroutine 37 [chan receive, 45 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher() /go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:100 +0x14a created by github.com/cockroachdb/cockroach/pkg/util/log.init.7 /go/src/github.com/cockroachdb/cockroach/pkg/util/log/log_flush.go:46 +0x5b goroutine 68327 [chan send, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration.func4(0x7173da0, 0xc000110010, 0xc00181a100, 0xc, 0xc, 0xc00035afc0, 0xc00095f510, 0xc000869800, 0x9) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1474 +0x329 created by github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1453 +0xce6 goroutine 10 [select]: github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view.(*worker).start(0xc000138400) /go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:152 +0x1b3 created by github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view.init.0 /go/src/github.com/cockroachdb/cockroach/vendor/go.opencensus.io/stats/view/worker.go:29 +0x9a goroutine 67500 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174f4) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc001713e60) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc001713e60) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f4ea0, 0xc001bd2be0, 0xc0002f4e90) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67591 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*NodeLiveness).StartHeartbeat.func1(0x7173e20, 0xc001567710) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/node_liveness.go:469 +0x3ed github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc005832970, 0xc001bd2be0, 0xc005ce9a40) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67661 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc0001307e0, 0x2, 0xc003d00780, 0xc006bbc200, 0x71cf8a0, 0xc005832a10, 0x71cf801, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:485 +0x74e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2(0x7173e20, 0xc0009256b0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:636 +0x436 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0032f2230, 0xc000e2e500, 0xc0068fd440) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67429 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func4(0x7173e20, 0xc004096420) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1639 +0x451 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c8d0, 0xc000e2e780, 0xc00782a000) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67602 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc000641620) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc000e94080, 0xc001bd2be0, 0xc006a62160) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67382 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d11) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc0034121b0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0034121b0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862360, 0xc000e2e780, 0xc004862350) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 68326 [chan send, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration.func4(0x7173da0, 0xc000110010, 0xc00181a100, 0xc, 0xc, 0xc00035afc0, 0xc00095f510, 0xc000869800, 0x6) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1474 +0x329 created by github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1453 +0xce6 goroutine 67340 [chan receive, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func1(0x7173e20, 0xc0034122a0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:156 +0x63 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862010, 0xc000e2e780, 0xc000f3e580) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67347 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d0f) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc003412420) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc003412420) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048620f0, 0xc000e2e780, 0xc0048620e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67584 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0059cf7c0, 0x1, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:317 +0x1fb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000efdaa0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x2ad github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc000bdc000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:330 +0x12b created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:328 +0x17ba goroutine 67396 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d1d) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000b62c60) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000b62c60) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862520, 0xc000e2e780, 0xc004862510) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67341 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d33) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc0034122d0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0034122d0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862030, 0xc000e2e780, 0xc004862020) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67246 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d02) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc003412450) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc003412450) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c0a0, 0xc000e2e780, 0xc00103c090) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67419 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d20) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc004096240) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004096240) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c450, 0xc000e2e780, 0xc00103c440) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67561 [semacquire, 42 minutes]: sync.runtime_Semacquire(0xc001710620) /usr/local/go/src/runtime/sema.go:56 +0x42 sync.(*WaitGroup).Wait(0xc001710618) /usr/local/go/src/sync/waitgroup.go:130 +0xb1 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Wait(...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:172 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5840, 0xc001bd2be0, 0xc0002f5820) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67513 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174d7) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc000e5bf20) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e5bf20) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f50f0, 0xc001bd2be0, 0xc0002f50e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67540 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174d3) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc0067f31d0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0067f31d0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5530, 0xc001bd2be0, 0xc0002f5520) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67444 [select, 42 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc003940700) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:115 +0x203 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:106 +0x2b9 goroutine 67589 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x7173e20, 0xc002a49b60) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1372 +0x5fd github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0058328b0, 0xc000e2e500, 0xc005832880) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67527 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc00001750e) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc001567050) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc001567050) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f52d0, 0xc001bd2be0, 0xc0002f52c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67299 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0007742d8, 0x36e4) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0007742c8) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/storage.(*RocksDB).syncLoop(0xc0007741a0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/rocksdb.go:678 +0x81 created by github.com/cockroachdb/cockroach/pkg/storage.(*RocksDB).open /go/src/github.com/cockroachdb/cockroach/pkg/storage/rocksdb.go:665 +0x46d goroutine 67550 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174f5) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854b40) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854b40) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5690, 0xc001bd2be0, 0xc0002f5680) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67704 [select, 42 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000d50a00, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:267 +0x15d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:278 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc001340700, 0x5f7c140, 0xc0015c5700, 0xc000745b50, 0xc5af1f, 0xc000f42d1c) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:854 +0xf8e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001340700, 0xc003477428, 0x179f530) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:719 +0x62 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).withRetry(0xc000f42c60, 0xc003477550, 0xc003477520, 0xc0034774d8, 0xc003477540) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:573 +0x43a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc000f42c60, 0x5f7c140, 0xc0015c5700, 0xc001154600, 0xed628558d) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:718 +0x168 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc00103cf00, 0x0, 0x0, 0xc00181eb70) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:75 +0x85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1.1(0x71cf8a0, 0xc00103cf00, 0xc00002cb40, 0xc0001307e0, 0x7173e20, 0xc00181eb70, 0xd34f7b, 0xc00181eb90) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:459 +0x5a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1(0x7173e20, 0xc00181eb70) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:474 +0x7e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103cf30, 0xc000e2e500, 0xc003547f80) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67460 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc0033aff40, 0xc003267870, 0x5, 0x5, 0x0, 0x0, 0xc000376a40) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:146 +0x178 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc0033aff40, 0xc003267870, 0x5, 0x5, 0xc000bfb458, 0x15ee910, 0xc000376a40) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:140 +0x31f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0067f3a40, 0xc003267870, 0x5, 0x5, 0xc000bfb488, 0x1604659, 0xc00352e780) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:435 +0x73 io.ReadAtLeast(0x70f1100, 0xc0067f3a40, 0xc003267870, 0x5, 0x5, 0x5, 0xbcf27a, 0xc0003a5750, 0xc0003a5750) /usr/local/go/src/io/io.go:310 +0x99 io.ReadFull(...) /usr/local/go/src/io/io.go:329 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0007c7200, 0xc003267870, 0x5, 0x5, 0x5af02c0, 0xc000bfb6c8, 0xc0018dd970) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:419 +0x17d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc003267860, 0x7fffffff, 0x259b89f, 0xc005aa6010, 0x3b, 0xe2e618, 0xc00054a300, 0xc005aa6000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:508 +0x87 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvAndDecompress(0xc003267860, 0xc0007c7200, 0x0, 0x0, 0x7fffffff, 0xc000bfb9b8, 0x715bda0, 0xa61e5f8, 0x24, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:639 +0x61 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc003267860, 0x7f595172e140, 0xc0002b5df0, 0xc0007c7200, 0x0, 0x0, 0x5f798c0, 0xc0004d8280, 0x7fffffff, 0xc000bfb9b8, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:684 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc000ce2000, 0x5f798c0, 0xc0004d8280, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:1458 +0x2b1 github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipServer).Recv(0xc0032f28c0, 0xc0000000, 0xc0014f8b54, 0xc000bfba70) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:588 +0x85 github.com/cockroachdb/cockroach/pkg/gossip.(*server).gossipReceiver(0xc0014f8b00, 0x7173e20, 0xc000a31680, 0xc0004ce1d0, 0xc00467cfc0, 0xc000bfbec0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:334 +0x5d1 github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip.func3.1(0x7173e20, 0xc000a31680) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:141 +0xb9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0032f2900, 0xc000e2e500, 0xc00467d000) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67575 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/compactor.(*Compactor).Start.func1.1(0x7173e20, 0xc001566a80) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/compactor/compactor.go:123 +0x278 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862ca0, 0xc001bd2be0, 0xc000f3f820) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67321 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc0040968a0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc003da00f0, 0xc000e2e780, 0xc000330460) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67545 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174db) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc000e67650) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e67650) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f55f0, 0xc001bd2be0, 0xc0002f55e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67538 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174e6) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc0067f31a0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0067f31a0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f54e0, 0xc001bd2be0, 0xc0002f54d0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67324 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc004096930) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc003da0120, 0xc000e2e780, 0xc000330500) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67378 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d31) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc003412090) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc003412090) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048622e0, 0xc000e2e780, 0xc0048622d0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67528 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc00001750d) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc001567080) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc001567080) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f52f0, 0xc001bd2be0, 0xc0002f52e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67381 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d09) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc003412120) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc003412120) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862340, 0xc000e2e780, 0xc004862330) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67422 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d32) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc0040962d0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0040962d0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c510, 0xc000e2e780, 0xc00103c500) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67526 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc000017507) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854ab0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854ab0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f52b0, 0xc001bd2be0, 0xc0002f52a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67512 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc000017501) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc002a492f0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc002a492f0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f50d0, 0xc001bd2be0, 0xc0002f50c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67339 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc000307320, 0x7173da0, 0xc000110010) /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:464 +0x47e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000e2e780, 0x7173da0, 0xc000110010, 0xc0002ed320, 0x28, 0x0, 0x0, 0xc00027f320) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:323 +0x163 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:318 +0x14b goroutine 67580 [chan receive, 42 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc000bc4000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1107 +0x86e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:767 +0xfc goroutine 67405 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00352e000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:935 +0x34d created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:282 +0x180e goroutine 67447 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000940000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1288 +0x1cb created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:281 +0x1f97 goroutine 67393 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d26) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000b62b70) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000b62b70) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048624c0, 0xc000e2e780, 0xc0048624b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67449 [runnable]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00467c540, 0x1, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:317 +0x1fb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0019c6c00, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x2ad github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc000940000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:330 +0x12b created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:328 +0x17ba goroutine 67604 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).waitAndProcess(0xc0014c92b0, 0x7173e20, 0xc001712030, 0xc001bd2be0, 0x36e6beac, 0xed6285eec, 0x0, 0xc000912000, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:217 +0x275 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1.1(0xc000912000, 0x18) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:279 +0x13c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc006854270, 0xc005cc54c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:366 +0x2d7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1(0x7173e20, 0xc001712030) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:277 +0x3bf github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc000e940d0, 0xc001bd2be0, 0xc006a621c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67329 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).waitAndProcess(0xc00054a680, 0x7173e20, 0xc0040969f0, 0xc000e2e780, 0x328e836c, 0xed6285eec, 0x0, 0xc000705800, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:217 +0x275 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1.1(0xc000705800, 0x18) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:279 +0x13c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*storeReplicaVisitor).Visit(0xc0009ae960, 0xc003941900) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:366 +0x2d7 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*replicaScanner).scanLoop.func1(0x7173e20, 0xc0040969f0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scanner.go:277 +0x3bf github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc003da0170, 0xc000e2e780, 0xc000330600) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67543 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174ec) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc002a494d0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc002a494d0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f55b0, 0xc001bd2be0, 0xc0002f55a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67573 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startClosedTimestampRangefeedSubscriber.func2(0x7173e20, 0xc002a493b0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1717 +0x3c7 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862c80, 0xc001bd2be0, 0xc000f3f7e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67643 [IO wait]: internal/poll.runtime_pollWait(0x7f5939038c38, 0x72, 0x70f4160) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(*pollDesc).wait(0xc00332d398, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0xe4 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc00332d380, 0xc000d00000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x253 net.(*netFD).Read(0xc00332d380, 0xc000d00000, 0x8000, 0x8000, 0x36, 0x0, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x66 net.(*conn).Read(0xc000ac8070, 0xc000d00000, 0x8000, 0x8000, 0xc001438b18, 0xc2b500, 0xc00078c780) /usr/local/go/src/net/net.go:184 +0xa1 bufio.(*Reader).Read(0xc000a248a0, 0xc000302038, 0x9, 0x9, 0xc0056161b0, 0x15e2e1a, 0xc0056161b0) /usr/local/go/src/bufio/bufio.go:226 +0x827 io.ReadAtLeast(0x70eb9e0, 0xc000a248a0, 0xc000302038, 0x9, 0x9, 0x9, 0x2d, 0x2d, 0x0) /usr/local/go/src/io/io.go:310 +0x99 io.ReadFull(...) /usr/local/go/src/io/io.go:329 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc000302038, 0x9, 0x9, 0x70eb9e0, 0xc000a248a0, 0x0, 0x0, 0x2d00000001, 0x10006e1a7) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xac github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc000302000, 0xc00438ef00, 0xc00438ef00, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xfc github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00035b180) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1237 +0x252 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:286 +0x1373 goroutine 67530 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174e3) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc002a49530) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc002a49530) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5350, 0xc001bd2be0, 0xc0002f5330) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67452 [IO wait]: internal/poll.runtime_pollWait(0x7f5951787e78, 0x72, 0x70f4160) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(*pollDesc).wait(0xc0004d6498, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0xe4 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc0004d6480, 0xc000d48000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x253 net.(*netFD).Read(0xc0004d6480, 0xc000d48000, 0x8000, 0x8000, 0x66, 0x0, 0x0) /usr/local/go/src/net/fd_unix.go:202 +0x66 net.(*conn).Read(0xc000ac8040, 0xc000d48000, 0x8000, 0x8000, 0xbd22ba, 0xc003d01800, 0xc00536ce40) /usr/local/go/src/net/net.go:184 +0xa1 bufio.(*Reader).Read(0xc000efc9c0, 0xc0007a01f8, 0x9, 0x9, 0xc0016112c8, 0xc002bbc960, 0x5d) /usr/local/go/src/bufio/bufio.go:226 +0x827 io.ReadAtLeast(0x70eb9e0, 0xc000efc9c0, 0xc0007a01f8, 0x9, 0x9, 0x9, 0x1, 0x5d, 0x5d) /usr/local/go/src/io/io.go:310 +0x99 io.ReadFull(...) /usr/local/go/src/io/io.go:329 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc0007a01f8, 0x9, 0x9, 0x70eb9e0, 0xc000efc9c0, 0x0, 0x0, 0xc001713470, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xac github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc0007a01c0, 0xc001713470, 0xc001713470, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xfc github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00352e780, 0xc0067f2450, 0x62a8e28) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:431 +0xeb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc003e90480, 0x71d6920, 0xc00352e780) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:713 +0x19b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func1(0xc003e90480, 0x71d6920, 0xc00352e780) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:675 +0x51 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:674 +0x783 goroutine 67312 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x7173e20, 0xc000b636e0) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1372 +0x5fd github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0032f20f0, 0xc000e2e500, 0xc0032f20e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67420 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d30) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc0040962a0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0040962a0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c4a0, 0xc000e2e780, 0xc00103c490) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67435 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startClosedTimestampRangefeedSubscriber.func2(0x7173e20, 0xc0040963c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1717 +0x3c7 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103ca90, 0xc000e2e780, 0xc001dc0060) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67326 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc004096960) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc003da0140, 0xc000e2e780, 0xc000330560) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67404 [runnable]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0059ce880, 0x1, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:317 +0x1fb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000efc120, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x2ad github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00352e000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:276 +0x1ec created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:273 +0x17e9 goroutine 67328 [select, 32 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc004096870) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc003da0160, 0xc000e2e780, 0xc0003305e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67555 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174fb) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854c30) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854c30) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5730, 0xc001bd2be0, 0xc0002f5720) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67494 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc00120a900, 0x7173da0, 0xc000110010) /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:464 +0x47e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc001bd2be0, 0x7173da0, 0xc000110010, 0xc0009c2840, 0x22, 0x0, 0x0, 0xc0002f4d50) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:323 +0x163 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:318 +0x14b goroutine 67524 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174ea) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854a80) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854a80) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5270, 0xc001bd2be0, 0xc0002f5250) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67387 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d1b) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000e66000) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e66000) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862400, 0xc000e2e780, 0xc0048623f0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67371 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d1a) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000b62960) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000b62960) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862200, 0xc000e2e780, 0xc0048621f0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67559 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174e9) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854cf0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854cf0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f57e0, 0xc001bd2be0, 0xc0002f57d0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67515 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174f0) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc002a49440) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc002a49440) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5130, 0xc001bd2be0, 0xc0002f5120) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67439 [IO wait, 42 minutes]: internal/poll.runtime_pollWait(0x7f5951743a38, 0x72, 0x72) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(*pollDesc).wait(0xc0019e4018, 0x72, 0x0, 0x0, 0x6111b3c) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0xe4 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Accept(0xc0019e4000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:384 +0x2cc net.(*netFD).accept(0xc0019e4000, 0x7f593620dcc8, 0x0, 0x0) /usr/local/go/src/net/fd_unix.go:238 +0x56 net.(*TCPListener).accept(0xc001dc0200, 0xc000ac8040, 0x0, 0x0) /usr/local/go/src/net/tcpsock_posix.go:139 +0x50 net.(*TCPListener).Accept(0xc001dc0200, 0x62a8dd0, 0xc003e90480, 0x71d04a0, 0xc000ac8040) /usr/local/go/src/net/tcpsock.go:261 +0x50 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve(0xc003e90480, 0x715b0e0, 0xc001dc0200, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:587 +0x316 github.com/cockroachdb/cockroach/pkg/util/netutil.ListenAndServeGRPC.func2(0x7173e20, 0xc002a48d50) /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:52 +0x54 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103cc20, 0xc000e2e500, 0xc001dc0220) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67436 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).Start.func1(0x7173e20, 0xc004096390) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_rebalancer.go:186 +0x213 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103cad0, 0xc000e2e780, 0xc001dc0080) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67313 [select]: github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x7173e20, 0xc000b63680) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1407 +0x5a7 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0032f2110, 0xc000e2e500, 0xc0032f2100) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67564 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Start.func2(0x7173e20, 0xc001472cc0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1498 +0x159 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5a10, 0xc001bd2be0, 0xc0003a3360) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67507 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174da) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc004097b60) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004097b60) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f4fc0, 0xc001bd2be0, 0xc0002f4fa0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67552 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174f3) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854c00) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854c00) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f56d0, 0xc001bd2be0, 0xc0002f56c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67457 [IO wait]: internal/poll.runtime_pollWait(0x7f594c22c218, 0x72, 0x70f4160) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(*pollDesc).wait(0xc002e56418, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0xe4 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc002e56400, 0xc00104e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x253 net.(*netFD).Read(0xc002e56400, 0xc00104e000, 0x8000, 0x8000, 0x203000, 0x7173e20, 0x1) /usr/local/go/src/net/fd_unix.go:202 +0x66 net.(*conn).Read(0xc0004ce1b0, 0xc00104e000, 0x8000, 0x8000, 0xc003f278f0, 0x9c550c0, 0x15fcd0d) /usr/local/go/src/net/net.go:184 +0xa1 bufio.(*Reader).Read(0xc005e056e0, 0xc00092a118, 0x9, 0x9, 0xc0059cf348, 0x0, 0x0) /usr/local/go/src/bufio/bufio.go:226 +0x827 io.ReadAtLeast(0x70eb9e0, 0xc005e056e0, 0xc00092a118, 0x9, 0x9, 0x9, 0x1, 0x43, 0x43) /usr/local/go/src/io/io.go:310 +0x99 io.ReadFull(...) /usr/local/go/src/io/io.go:329 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc00092a118, 0x9, 0x9, 0x70eb9e0, 0xc005e056e0, 0x0, 0x0, 0xc0067f3290, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xac github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc00092a0e0, 0xc0067f3290, 0xc0067f3290, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xfc github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc001a11200, 0xc0067f33b0, 0x62a8e28) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:431 +0xeb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc0047e7380, 0x71d6920, 0xc001a11200) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:713 +0x19b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0047e7380, 0x71d6920, 0xc001a11200) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:675 +0x51 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:674 +0x783 goroutine 67249 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d06) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc002a48cf0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc002a48cf0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c110, 0xc000e2e780, 0xc00103c100) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67503 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174fd) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc000e5bdd0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e5bdd0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f4f00, 0xc001bd2be0, 0xc0002f4ef0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67336 [chan receive, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration(0xc000852f00) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1485 +0xd37 testing.tRunner(0xc000852f00, 0x62a3870) /usr/local/go/src/testing/testing.go:909 +0x19a created by testing.(*T).Run /usr/local/go/src/testing/testing.go:960 +0x652 goroutine 67598 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc000925260) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc000e94030, 0xc001bd2be0, 0xc006a620e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67674 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc005b1af00, 0xc001feb290, 0x5, 0x5, 0x0, 0x0, 0xc0003388a0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:158 +0x186 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc005b1af00, 0xc001feb290, 0x5, 0x5, 0xc000d76c28, 0x15e2ec0, 0xc0003388a0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:138 +0x222 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0038739b0, 0xc001feb290, 0x5, 0x5, 0xc000d76c58, 0x1602269, 0xc00035b180) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:435 +0x73 io.ReadAtLeast(0x70f1100, 0xc0038739b0, 0xc001feb290, 0x5, 0x5, 0x5, 0x12, 0x14, 0x0) /usr/local/go/src/io/io.go:310 +0x99 io.ReadFull(...) /usr/local/go/src/io/io.go:329 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00081d700, 0xc001feb290, 0x5, 0x5, 0xc0015c50a8, 0xc000d76d50, 0xbd784a) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:419 +0x17d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc001feb280, 0x7fffffff, 0xc000d76e28, 0x160e416, 0xc0006fa600, 0x16, 0x600, 0x5f805e0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:508 +0x87 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvAndDecompress(0xc001feb280, 0xc00081d700, 0x0, 0x0, 0x7fffffff, 0x0, 0x715bda0, 0xa61e5f8, 0xc0006fa600, 0xc00390c000, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:639 +0x61 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc001feb280, 0x7f595172e140, 0xc0002b5df0, 0xc00081d700, 0x0, 0x0, 0x5f805e0, 0xc0015c5080, 0x7fffffff, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:684 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc001604480, 0x5f805e0, 0xc0015c5080, 0xc000d771d0, 0xc5af1f, 0xc000dec89c) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:905 +0x7ea github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001604480, 0x2c, 0x2c) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:719 +0x62 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).withRetry(0xc000dec7e0, 0xc000d773d0, 0xc000d773a0, 0xc000dec890, 0xc005b1b030) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:577 +0xe2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc000dec7e0, 0x5f805e0, 0xc0015c5080, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:718 +0x168 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7173d60, 0xc001610140, 0x615060a, 0x1d, 0x5f804c0, 0xc000c722a0, 0x5f805e0, 0xc0015c5080, 0xc001c66000, 0xc00175ae80, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:73 +0x195 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x7173d60, 0xc001610140, 0x615060a, 0x1d, 0x5f804c0, 0xc000c722a0, 0x5f805e0, 0xc0015c5080, 0xc001c66000, 0x62a8ec0, ...) /go/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:47 +0xccb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc001c66000, 0x7173d60, 0xc001610140, 0x615060a, 0x1d, 0x5f804c0, 0xc000c722a0, 0x5f805e0, 0xc0015c5080, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:35 +0x193 github.com/cockroachdb/cockroach/pkg/rpc.(*heartbeatClient).Ping(0xc0004ce098, 0x7173d60, 0xc001610140, 0xc000c722a0, 0x0, 0x0, 0x0, 0x6b49d27d, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/heartbeat.pb.go:196 +0x106 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3.1(...) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1090 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3(0x7173d60, 0xc001610140, 0xd, 0xc0008e1a01) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1097 +0xddc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000e2e500, 0x7173d60, 0xc001610140, 0x6120f23, 0xd, 0xc000d77d50, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:302 +0x151 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc0016b2000, 0xc007775540, 0xc0003a4c90, 0xf, 0xc004a8a6c0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1074 +0x624 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1.1(0x7173e20, 0xc00166b140) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1000 +0x95 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc005832540, 0xc000e2e500, 0xc007775590) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67406 [runnable]: syscall.Syscall(0x0, 0x8, 0xc0008a8000, 0x8000, 0xffffffffffffffff, 0x0, 0xb) /usr/local/go/src/syscall/asm_linux_amd64.s:18 +0x5 syscall.read(0x8, 0xc0008a8000, 0x8000, 0x8000, 0x7f5949bd2160, 0x72, 0x0) /usr/local/go/src/syscall/zsyscall_linux_amd64.go:732 +0x68 syscall.Read(0x8, 0xc0008a8000, 0x8000, 0x8000, 0x0, 0x0, 0xc067de) /usr/local/go/src/syscall/syscall_unix.go:183 +0x5b internal/poll.(*FD).Read(0xc001696e00, 0xc0008a8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:165 +0x1c8 net.(*netFD).Read(0xc001696e00, 0xc0008a8000, 0x8000, 0x8000, 0x203000, 0x7173e20, 0x1) /usr/local/go/src/net/fd_unix.go:202 +0x66 net.(*conn).Read(0xc003362080, 0xc0008a8000, 0x8000, 0x8000, 0xc0035473b0, 0xc00043d180, 0x15fcd0d) /usr/local/go/src/net/net.go:184 +0xa1 bufio.(*Reader).Read(0xc000efc060, 0xc0007a0118, 0x9, 0x9, 0xc00098df48, 0x0, 0x0) /usr/local/go/src/bufio/bufio.go:226 +0x827 io.ReadAtLeast(0x70eb9e0, 0xc000efc060, 0xc0007a0118, 0x9, 0x9, 0x9, 0x1, 0x43, 0x43) /usr/local/go/src/io/io.go:310 +0x99 io.ReadFull(...) /usr/local/go/src/io/io.go:329 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc0007a0118, 0x9, 0x9, 0x70eb9e0, 0xc000efc060, 0x0, 0x0, 0xc001712570, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xac github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc0007a00e0, 0xc001712570, 0xc001712570, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xfc github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00352e000, 0xc001712690, 0x62a8e28) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:431 +0xeb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc003e90480, 0x71d6920, 0xc00352e000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:713 +0x19b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func1(0xc003e90480, 0x71d6920, 0xc00352e000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:675 +0x51 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:674 +0x783 goroutine 67497 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174e8) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc000e5bd40) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e5bd40) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f4dc0, 0xc001bd2be0, 0xc0002f4db0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 68341 [chan send, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration.func4(0x7173da0, 0xc000110010, 0xc00181a100, 0xc, 0xc, 0xc00035afc0, 0xc00095f510, 0xc000869800, 0x8) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1474 +0x329 created by github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1453 +0xce6 goroutine 67323 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc004096900) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc003da0110, 0xc000e2e780, 0xc0003304e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67345 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d12) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc0034123c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0034123c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048620b0, 0xc000e2e780, 0xc0048620a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67517 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174e5) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc0068548d0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0068548d0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5170, 0xc001bd2be0, 0xc0002f5160) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67337 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc000306900, 0x7173da0, 0xc000110010) /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:464 +0x47e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000e2e780, 0x7173da0, 0xc000110010, 0xc000309c50, 0x22, 0x0, 0x0, 0xc00027f290) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:323 +0x163 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:318 +0x14b goroutine 67431 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func4(0x7173e20, 0xc003412510) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1639 +0x451 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c950, 0xc000e2e780, 0xc00782a0a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67369 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d3f) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000eb8030) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000eb8030) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048621c0, 0xc000e2e780, 0xc0048621a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67384 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d3d) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc003412000) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc003412000) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048623a0, 0xc000e2e780, 0xc004862390) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67511 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174e1) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc004097bf0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004097bf0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f50b0, 0xc001bd2be0, 0xc0002f50a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67372 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d38) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000e5a960) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e5a960) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862220, 0xc000e2e780, 0xc004862210) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67551 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174ee) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854ba0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854ba0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f56b0, 0xc001bd2be0, 0xc0002f56a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67534 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174ed) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc001a32060) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc001a32060) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5420, 0xc001bd2be0, 0xc0002f5400) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67779 [chan receive, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration.func1(0x7173e20, 0xc001567f50, 0x7b, 0x143, 0x100000001, 0x1, 0x0, 0x1, 0x0, 0xc0006dd0e0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1346 +0x143 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithRangeID(0xc000b5a800, 0x7173e20, 0xc001567f50, 0x1, 0xc00332c480, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:84 +0xb28 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc000b5a800, 0x7173e20, 0xc001567f50, 0x7b, 0x143, 0x100000001, 0x1, 0x0, 0x1, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:36 +0xce github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc000798000, 0x7173e20, 0xc001567f50, 0x7b, 0x143, 0x100000001, 0x1, 0x0, 0x1, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:202 +0x6c4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0004fa000, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x1, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:177 +0x206 github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.(*multiTestContextKVTransport).SendNext.func1(0x7173e20, 0xc001567f50) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_test.go:596 +0xd7 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000e2e780, 0x7173e20, 0xc001567f50, 0x6114cfa, 0x8, 0xc000c97730, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:285 +0x151 github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.(*multiTestContextKVTransport).SendNext(0xc003940e40, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x1, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_test.go:595 +0x606 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc000130b40, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1638 +0x348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendRPC(0xc000130b40, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:456 +0x32c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendSingleRange(0xc000130b40, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:538 +0x3e1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc000130b40, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1407 +0x4a0 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc000130b40, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1079 +0x1d90 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc000130b40, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:730 +0xa99 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc001206450, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x195 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc001206418, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0xf4 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc0012063e8, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:195 +0x1e7 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc001206350, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:247 +0x108 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc001206350, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:182 +0x2cb github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc001206290, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:252 +0x1ea github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc001206270, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x2c3 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc0012061d0, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:172 +0xff github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc001206000, 0x7173e20, 0xc001567f50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:503 +0x6b7 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc001604380, 0x7173e20, 0xc001a25920, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:739 +0x174 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc000e68ab0, 0x7173e20, 0xc001a25920, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:907 +0x1d2 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x7173e20, 0xc001a25920, 0xc000c9bec0, 0xc00068e000, 0xc00068e000, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:657 +0x165 github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Run(0xc000e68ab0, 0x7173e20, 0xc001a25920, 0xc00068e000, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:575 +0x120 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).AdminMerge.func1(0xc000e68ab0, 0xc001a25920, 0xc001604380) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_command.go:752 +0x1a45 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).AdminMerge(0xc000b5a800, 0x7173e20, 0xc001a25920, 0xa61e5f8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_command.go:771 +0x1ec github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeAdminBatch(0xc000b5a800, 0x7173e20, 0xc001a25920, 0xc0019e4180, 0xa, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:466 +0xe60 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithRangeID(0xc000b5a800, 0x7173e20, 0xc001a258f0, 0x1, 0xc0019e4180, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:101 +0x8de github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc000b5a800, 0x7173e20, 0xc0009af260, 0x7b, 0x140, 0x100000001, 0x1, 0x0, 0x1, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:36 +0xce github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc000798000, 0x7173da0, 0xc000110010, 0x7b, 0x140, 0x100000001, 0x1, 0x0, 0x1, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:202 +0x6c4 github.com/cockroachdb/cockroach/pkg/kv.Wrap.func1(0x7173da0, 0xc000110010, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/sender.go:426 +0x109 github.com/cockroachdb/cockroach/pkg/kv.SenderFunc.Send(0xc0043bcfe0, 0x7173da0, 0xc000110010, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/sender.go:362 +0xac github.com/cockroachdb/cockroach/pkg/kv.SendWrappedWith(0x7173da0, 0xc000110010, 0x70f2040, 0xc0043bcfe0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/sender.go:402 +0x166 github.com/cockroachdb/cockroach/pkg/kv.SendWrapped(...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/sender.go:419 github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration.func3(0xc00095f4a0, 0x7173da0, 0xc000110010, 0xc00035afc0, 0xc000978420) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1397 +0x1f1 created by github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1395 +0x831 goroutine 67441 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000dfe900, 0xc000f0d050, 0xc14e9f) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:267 +0x15d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:278 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc002e56000, 0x5f805e0, 0xc004474600, 0xc000f0d1d0, 0xc5af1f, 0xc000130d1c) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:854 +0xf8e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc002e56000, 0x2c, 0x2c) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:719 +0x62 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).withRetry(0xc000130c60, 0xc000f0d3d0, 0xc000f0d3a0, 0xc000130d10, 0xc006721210) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:577 +0xe2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc000130c60, 0x5f805e0, 0xc004474600, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:718 +0x168 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7173d60, 0xc001610140, 0x615060a, 0x1d, 0x5f804c0, 0xc004e47080, 0x5f805e0, 0xc004474600, 0xc00103a700, 0xc003940100, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:73 +0x195 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x7173d60, 0xc001610140, 0x615060a, 0x1d, 0x5f804c0, 0xc004e47080, 0x5f805e0, 0xc004474600, 0xc00103a700, 0x62a8ec0, ...) /go/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:47 +0xccb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc00103a700, 0x7173d60, 0xc001610140, 0x615060a, 0x1d, 0x5f804c0, 0xc004e47080, 0x5f805e0, 0xc004474600, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:35 +0x193 github.com/cockroachdb/cockroach/pkg/rpc.(*heartbeatClient).Ping(0xc000498550, 0x7173d60, 0xc001610140, 0xc004e47080, 0x0, 0x0, 0x0, 0x6b49d27d, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/heartbeat.pb.go:196 +0x106 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3.1(...) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1090 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3(0x7173d60, 0xc001610140, 0xd, 0xc000a43b01) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1097 +0xddc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000e2e500, 0x7173d60, 0xc001610140, 0x6120f23, 0xd, 0xc000f0dd50, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:302 +0x151 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc0016b2000, 0xc00782a0f0, 0xc0031ec050, 0xf, 0xc0040781e0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1074 +0x624 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1.1(0x7173e20, 0xc002a48cc0) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1000 +0x95 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103cd80, 0xc000e2e500, 0xc00782a140) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67541 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc000017511) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc0067f3200) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0067f3200) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5560, 0xc001bd2be0, 0xc0002f5550) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67615 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc0001307e0, 0x71cf900, 0xc00103cd50, 0x712e520, 0xc0001307e0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:384 +0x26b github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler(0x6066cc0, 0xc0001307e0, 0x71b4840, 0xc000ce20c0, 0xc0004c4868, 0xc000a84700) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:123 +0xce github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc0047e7380, 0x71d6920, 0xc00352e180, 0xc000a84700, 0xc003412ed0, 0x995cd20, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1203 +0x1658 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc0047e7380, 0x71d6920, 0xc00352e180, 0xc000a84700, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1283 +0x12d8 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc00033af90, 0xc0047e7380, 0x71d6920, 0xc00352e180, 0xc000a84700) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:717 +0xc9 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:715 +0xb9 goroutine 67491 [runnable]: sync.(*Mutex).Unlock(0xc001610990) /usr/local/go/src/sync/mutex.go:179 +0x82 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc001610980, 0x1, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:308 +0x31e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000fa92c0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x2ad github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00352e180) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:276 +0x1ec created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:273 +0x17e9 goroutine 67415 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d0c) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc004096180) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004096180) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c340, 0xc000e2e780, 0xc00103c330) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67510 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc00001750c) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc001713fb0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc001713fb0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5040, 0xc001bd2be0, 0xc0002f5030) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67522 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc000017510) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854930) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854930) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5210, 0xc001bd2be0, 0xc0002f5200) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67284 [chan receive, 42 minutes]: github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x7173e20, 0xc002a49cb0) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:383 +0x85 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc005832870, 0xc000e2e500, 0xc00330f140) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67516 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174e2) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc000e675c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e675c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5150, 0xc001bd2be0, 0xc0002f5140) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67665 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch(0xc0001307e0, 0x71cf900, 0xc0018dca40, 0x712e520, 0xc0001307e0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:384 +0x26b github.com/cockroachdb/cockroach/pkg/kv/kvserver._MultiRaft_RaftMessageBatch_Handler(0x6066cc0, 0xc0001307e0, 0x71b4840, 0xc0019b29c0, 0xc0004c4868, 0xc00160e500) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:123 +0xce github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc003e90480, 0x71d6920, 0xc00352e780, 0xc00160e500, 0xc0008b6c00, 0x995cd20, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1203 +0x1658 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc003e90480, 0x71d6920, 0xc00352e780, 0xc00160e500, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1283 +0x12d8 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc00033ad80, 0xc003e90480, 0x71d6920, 0xc00352e780, 0xc00160e500) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:717 +0xc9 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:715 +0xb9 goroutine 67367 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d28) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000e660f0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e660f0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862170, 0xc000e2e780, 0xc004862160) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67549 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174e0) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854b10) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854b10) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5670, 0xc001bd2be0, 0xc0002f5660) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67582 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc000bdc000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1288 +0x1cb created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:281 +0x1f97 goroutine 67531 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174ef) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc001a32090) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc001a32090) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5380, 0xc001bd2be0, 0xc0002f5370) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67558 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc00001750a) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854cc0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854cc0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f57b0, 0xc001bd2be0, 0xc0002f57a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67437 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/compactor.(*Compactor).Start.func1.1(0x7173e20, 0xc004096360) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/compactor/compactor.go:123 +0x278 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103cb10, 0xc000e2e780, 0xc001dc00a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67413 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d3b) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc004096120) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004096120) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c280, 0xc000e2e780, 0xc00103c210) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67450 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0059cef80, 0x1, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:317 +0x1fb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00536c360, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x2ad github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc00352e780) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:276 +0x1ec created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:273 +0x17e9 goroutine 67322 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc0040968d0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc003da0100, 0xc000e2e780, 0xc0003304c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67414 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d07) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc004096150) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004096150) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c320, 0xc000e2e780, 0xc00103c2d0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67155 [chan receive, 42 minutes]: github.com/cockroachdb/cockroach/pkg/rpc.NewContextWithTestingKnobs.func1(0x7173e20, 0xc0009ae000) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:475 +0x68 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0018dc0f0, 0xc000e2e500, 0xc0039282a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67380 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d08) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc0034120f0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0034120f0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862320, 0xc000e2e780, 0xc004862310) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67508 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc00001750b) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc004097b90) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004097b90) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f4fe0, 0xc001bd2be0, 0xc0002f4fd0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67370 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d1f) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000b624b0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000b624b0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048621e0, 0xc000e2e780, 0xc0048621d0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67252 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d04) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc0040960c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0040960c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c200, 0xc000e2e780, 0xc00103c1e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67394 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d0b) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000b62b40) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000b62b40) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048624e0, 0xc000e2e780, 0xc0048624d0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67523 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc000017502) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc0068549c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0068549c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5240, 0xc001bd2be0, 0xc0002f5230) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67603 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc001712060) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc000e940c0, 0xc001bd2be0, 0xc006a62180) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67554 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174de) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854c60) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854c60) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5710, 0xc001bd2be0, 0xc0002f5700) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67529 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174d6) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc000e67770) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e67770) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5320, 0xc001bd2be0, 0xc0002f5310) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 68228 [chan send, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration.func4(0x7173da0, 0xc000110010, 0xc00181a100, 0xc, 0xc, 0xc00035afc0, 0xc00095f510, 0xc000869800, 0x2) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1474 +0x329 created by github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1453 +0xce6 goroutine 67346 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d23) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000e662a0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e662a0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048620d0, 0xc000e2e780, 0xc0048620c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67311 [chan receive, 42 minutes]: github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x7173e20, 0xc000b636b0) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:383 +0x85 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0032f20d0, 0xc000e2e500, 0xc003266240) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67250 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d3a) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc004096090) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004096090) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c170, 0xc000e2e780, 0xc00103c120) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67374 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d24) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000b62c90) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000b62c90) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862260, 0xc000e2e780, 0xc004862250) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 68294 [chan send, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration.func4(0x7173da0, 0xc000110010, 0xc00181a100, 0xc, 0xc, 0xc00035afc0, 0xc00095f510, 0xc000869800, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1474 +0x329 created by github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1453 +0xce6 goroutine 68257 [chan send, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration.func4(0x7173da0, 0xc000110010, 0xc00181a100, 0xc, 0xc, 0xc00035afc0, 0xc00095f510, 0xc000869800, 0x4) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1474 +0x329 created by github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1453 +0xce6 goroutine 67417 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d22) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc0040961e0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0040961e0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c3e0, 0xc000e2e780, 0xc00103c3b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67343 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d0d) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc003412300) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc003412300) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862070, 0xc000e2e780, 0xc004862060) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67509 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc000017508) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc004097bc0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004097bc0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5010, 0xc001bd2be0, 0xc0002f5000) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67885 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait.(*Queue).MaybeWaitForPush(0xc003d7fce0, 0x7173e20, 0xc000e5a9f0, 0xc00000c5a0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/txnwait/queue.go:513 +0x12af github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).maybeInterceptReq(0xc0059ce180, 0x7173e20, 0xc000e5a9f0, 0x0, 0x7b, 0x175, 0x0, 0x0, 0xc001340000, 0x1, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:189 +0xf9 github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).sequenceReqWithGuard(0xc0059ce180, 0x7173e20, 0xc000e5a9f0, 0xc000509ea0, 0x0, 0x7b, 0x175, 0x0, 0x0, 0xc001340000, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:142 +0xec github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency.(*managerImpl).SequenceReq(0xc0059ce180, 0x7173e20, 0xc000e5a9f0, 0xc000509ea0, 0x0, 0x7b, 0x175, 0x0, 0x0, 0xc001340000, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/concurrency_manager.go:121 +0x1bc github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000b5a800, 0x7173e20, 0xc000e5a9f0, 0xc001340100, 0x62a1838, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:224 +0x48d github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithRangeID(0xc000b5a800, 0x7173e20, 0xc000e5a9f0, 0x1, 0xc001340100, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:94 +0xa8c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc000b5a800, 0x7173e20, 0xc000e5a9c0, 0x7b, 0x175, 0x100000001, 0x1, 0x0, 0x1, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:36 +0xce github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc000798000, 0x7173e20, 0xc000e5a8d0, 0x7b, 0x175, 0x100000001, 0x1, 0x0, 0x1, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:202 +0x6c4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc0004fa000, 0x7173e20, 0xc000e5a8d0, 0x7b, 0x175, 0x100000001, 0x1, 0x0, 0x1, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:177 +0x206 github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.(*multiTestContextKVTransport).SendNext.func1(0x7173e20, 0xc000e5a8d0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_test.go:596 +0xd7 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc000e2e780, 0x7173e20, 0xc000e5a8d0, 0x6114cfa, 0x8, 0xc000838ee0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:285 +0x151 github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.(*multiTestContextKVTransport).SendNext(0xc003940580, 0x7173e20, 0xc000e5a8d0, 0x7b, 0x175, 0x100000001, 0x1, 0x0, 0x1, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_test.go:595 +0x606 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc00120a360, 0x7173e20, 0xc000e5a8d0, 0x7b, 0x175, 0x0, 0x0, 0x0, 0x1, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1638 +0x348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendRPC(0xc00120a360, 0x7173e20, 0xc000e5a8d0, 0x7b, 0x175, 0x0, 0x0, 0x0, 0x1, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:456 +0x32c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendSingleRange(0xc00120a360, 0x7173e20, 0xc000e5a8d0, 0x7b, 0x175, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:538 +0x3e1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc00120a360, 0x7173e20, 0xc000e5a8d0, 0x7b, 0x175, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1407 +0x4a0 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc00120a360, 0x7173e20, 0xc000e5a8d0, 0x7b, 0x175, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1079 +0x1d90 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc00120a360, 0x7173e20, 0xc00131d440, 0x7b, 0x175, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:730 +0xa99 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc00332c760, 0x7173e20, 0xc00131d440, 0x7b, 0x175, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xe0 github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc00332c700, 0x7173e20, 0xc00131d440, 0x7b, 0x175, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:739 +0x174 github.com/cockroachdb/cockroach/pkg/kv.(*DB).send(...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:721 github.com/cockroachdb/cockroach/pkg/kv.sendAndFill(0x7173e20, 0xc00131d440, 0xc000c33870, 0xc000b68580, 0x1, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:657 +0x165 github.com/cockroachdb/cockroach/pkg/kv.(*DB).Run(0xc00332c700, 0x7173e20, 0xc00131d440, 0xc000b68580, 0x4000000000000000, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:680 +0xd2 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).maybeWatchForMerge.func1(0x7173e20, 0xc00131d440) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica.go:1333 +0x67e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc001bd2be0, 0x7173e20, 0xc00131d440, 0xc000d521c0, 0x16, 0x0, 0x0, 0xc001567230) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:323 +0x163 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:318 +0x14b goroutine 67535 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174f6) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc001a320c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc001a320c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5470, 0xc001bd2be0, 0xc0002f5430) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67544 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174f7) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc002a49500) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc002a49500) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f55d0, 0xc001bd2be0, 0xc0002f55c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67156 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0016b22d8, 0x5422) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0016b22c8) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/storage.(*RocksDB).syncLoop(0xc0016b21a0) /go/src/github.com/cockroachdb/cockroach/pkg/storage/rocksdb.go:678 +0x81 created by github.com/cockroachdb/cockroach/pkg/storage.(*RocksDB).open /go/src/github.com/cockroachdb/cockroach/pkg/storage/rocksdb.go:665 +0x46d goroutine 67525 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174fa) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854ae0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854ae0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5290, 0xc001bd2be0, 0xc0002f5280) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67518 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174d5) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854870) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854870) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5190, 0xc001bd2be0, 0xc0002f5180) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67701 [IO wait]: internal/poll.runtime_pollWait(0x7f594af193b0, 0x72, 0x70f4160) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(*pollDesc).wait(0xc0019e4718, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0xe4 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc0019e4700, 0xc000d54000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x253 net.(*netFD).Read(0xc0019e4700, 0xc000d54000, 0x8000, 0x8000, 0x203000, 0x7173e20, 0x1) /usr/local/go/src/net/fd_unix.go:202 +0x66 net.(*conn).Read(0xc002ffa088, 0xc000d54000, 0x8000, 0x8000, 0xc003873b90, 0xc0009f8700, 0x15fcd0d) /usr/local/go/src/net/net.go:184 +0xa1 bufio.(*Reader).Read(0xc003d00ae0, 0xc0001e8118, 0x9, 0x9, 0xc0015c5208, 0x0, 0x0) /usr/local/go/src/bufio/bufio.go:226 +0x827 io.ReadAtLeast(0x70eb9e0, 0xc003d00ae0, 0xc0001e8118, 0x9, 0x9, 0x9, 0x1, 0x43, 0x43) /usr/local/go/src/io/io.go:310 +0x99 io.ReadFull(...) /usr/local/go/src/io/io.go:329 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc0001e8118, 0x9, 0x9, 0x70eb9e0, 0xc003d00ae0, 0x0, 0x0, 0xc0068fda70, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xac github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc0001e80e0, 0xc0068fda70, 0xc0068fda70, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xfc github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc00352e180, 0xc0035476b0, 0x62a8e28) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:431 +0xeb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc0047e7380, 0x71d6920, 0xc00352e180) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:713 +0x19b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func1(0xc0047e7380, 0x71d6920, 0xc00352e180) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:675 +0x51 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:674 +0x783 goroutine 67338 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc000307200, 0x7173da0, 0xc000110010) /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:464 +0x47e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000e2e780, 0x7173da0, 0xc000110010, 0xc0002ed080, 0x22, 0x0, 0x0, 0xc00027f2a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:323 +0x163 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:318 +0x14b goroutine 67755 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc000306900, 0x7173da0, 0xc000110010, 0x2, 0x71f05c0, 0xc002d27860, 0x0, 0x1, 0xc10a5a, 0xc00088fc20) /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:248 +0x255 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0007fc000, 0x7173da0, 0xc000110010, 0x2, 0xc006ccb0e0, 0x18, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:700 +0x563 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2(0x7173da0, 0xc000110010) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:739 +0x91 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000e2e780, 0x7173da0, 0xc000110010, 0xc0004a7200, 0x33, 0x0, 0x0, 0xc000a235c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:323 +0x163 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:318 +0x14b goroutine 67590 [select]: github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x7173e20, 0xc001473710) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1407 +0x5a7 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0058328d0, 0xc000e2e500, 0xc0058328c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67642 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00035b180) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1288 +0x1cb created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:281 +0x1f97 goroutine 67425 [semacquire, 42 minutes]: sync.runtime_Semacquire(0xc00107a080) /usr/local/go/src/runtime/sema.go:56 +0x42 sync.(*WaitGroup).Wait(0xc00107a078) /usr/local/go/src/sync/waitgroup.go:130 +0xb1 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Wait(...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:172 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c680, 0xc000e2e780, 0xc00103c640) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67390 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d3e) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000b62a80) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000b62a80) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862460, 0xc000e2e780, 0xc004862450) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67553 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174f8) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854bd0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854bd0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f56f0, 0xc001bd2be0, 0xc0002f56e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67536 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174f9) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc001a32000) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc001a32000) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5490, 0xc001bd2be0, 0xc0002f5480) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67397 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d34) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc002a481b0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc002a481b0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862540, 0xc000e2e780, 0xc004862530) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67319 [IO wait]: internal/poll.runtime_pollWait(0x7f594af192e0, 0x72, 0x70f4160) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(*pollDesc).wait(0xc00332c298, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0xe4 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc00332c280, 0xc0007d6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x253 net.(*netFD).Read(0xc00332c280, 0xc0007d6000, 0x8000, 0x8000, 0x0, 0x0, 0xc002bbc478) /usr/local/go/src/net/fd_unix.go:202 +0x66 net.(*conn).Read(0xc000ac8000, 0xc0007d6000, 0x8000, 0x8000, 0x0, 0x10, 0xc00308c3c0) /usr/local/go/src/net/net.go:184 +0xa1 bufio.(*Reader).Read(0xc004e466c0, 0xc000806038, 0x9, 0x9, 0x15e5b6a, 0xc00082c000, 0xc000ce9ca1) /usr/local/go/src/bufio/bufio.go:226 +0x827 io.ReadAtLeast(0x70eb9e0, 0xc004e466c0, 0xc000806038, 0x9, 0x9, 0x9, 0x20, 0xc000ce9c60, 0x15e59ba) /usr/local/go/src/io/io.go:310 +0x99 io.ReadFull(...) /usr/local/go/src/io/io.go:329 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc000806038, 0x9, 0x9, 0x70eb9e0, 0xc004e466c0, 0x0, 0xc000000000, 0xc0007b8f00, 0xc00308c3b0) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xac github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc000806000, 0xc003547110, 0xc003547100, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xfc github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00082c000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1237 +0x252 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:286 +0x1373 goroutine 67348 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d2b) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc003412270) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc003412270) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862110, 0xc000e2e780, 0xc004862100) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67423 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d01) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc004096300) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004096300) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c580, 0xc000e2e780, 0xc00103c540) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67320 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00175a000, 0x1, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:317 +0x1fb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc004e46780, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x2ad github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc00082c000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:330 +0x12b created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:328 +0x17ba goroutine 67539 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc000017505) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc0067f3140) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0067f3140) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5510, 0xc001bd2be0, 0xc0002f54f0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67368 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d05) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000eb8720) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000eb8720) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862190, 0xc000e2e780, 0xc004862180) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67438 [chan receive, 42 minutes]: github.com/cockroachdb/cockroach/pkg/util/netutil.ListenAndServeGRPC.func1(0x7173e20, 0xc0040963f0) /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:45 +0x7a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103cc10, 0xc000e2e500, 0xc002a48ae0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67599 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc000eb8c00) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc000e94050, 0xc001bd2be0, 0xc006a62100) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67251 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d03) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc0040960f0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0040960f0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c1c0, 0xc000e2e780, 0xc00103c190) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67456 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc001a11200) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:935 +0x34d created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:282 +0x180e goroutine 67401 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d0a) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc0009afc20) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0009afc20) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048625c0, 0xc000e2e780, 0xc0048625b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67389 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d00) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc002a480c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc002a480c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862440, 0xc000e2e780, 0xc004862430) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67566 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func4(0x7173e20, 0xc006854d50) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1639 +0x451 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5a40, 0xc001bd2be0, 0xc00091bae0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67455 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00467cbc0, 0x1, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:317 +0x1fb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000efde00, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x2ad github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc001a11200) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:276 +0x1ec created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:273 +0x17e9 goroutine 67264 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).readClient(0xc00094a050, 0xc006a63990, 0x5, 0x5, 0x0, 0x0, 0xc002c862c0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:158 +0x186 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc00094a050, 0xc006a63990, 0x5, 0x5, 0xc00085d578, 0x15e2ec0, 0xc002c862c0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:138 +0x222 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0006415c0, 0xc006a63990, 0x5, 0x5, 0xc00085d5a8, 0x1602269, 0xc000940000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:435 +0x73 io.ReadAtLeast(0x70f1100, 0xc0006415c0, 0xc006a63990, 0x5, 0x5, 0x5, 0xc0000f0050, 0xc0000f0050, 0xc30432) /usr/local/go/src/io/io.go:310 +0x99 io.ReadFull(...) /usr/local/go/src/io/io.go:329 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*Stream).Read(0xc0008a0a00, 0xc006a63990, 0x5, 0x5, 0xc00085d740, 0xc00085d6a0, 0xbd784a) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:419 +0x17d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc006a63980, 0x7fffffff, 0xc00085d778, 0x160e416, 0xc0007ae600, 0x24, 0x600, 0x5f799e0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:508 +0x87 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvAndDecompress(0xc006a63980, 0xc0008a0a00, 0x0, 0x0, 0x7fffffff, 0xc00085db20, 0x715bda0, 0xa61e5f8, 0xc0007ae600, 0xc004e461e0, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:639 +0x61 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc006a63980, 0x7f595172e140, 0xc0002b5df0, 0xc0008a0a00, 0x0, 0x0, 0x5f799e0, 0xc006720690, 0x7fffffff, 0xc00085db20, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:684 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc001624480, 0x5f799e0, 0xc006720690, 0xc00085db20, 0xc5af1f, 0xc006ccad1c) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:868 +0x2ce github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc001624480, 0xc003413080, 0x1) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:719 +0x62 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).withRetry(0xc006ccac60, 0xc00085dd20, 0xc00085dcf0, 0xc00085de60, 0xc00085dd10) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:573 +0x43a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc006ccac60, 0x5f799e0, 0xc006720690, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:718 +0x168 github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc000e946d0, 0x7173e20, 0xc000641800, 0xc000774340) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:553 +0x85 github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0xc00334b680, 0x71cf780, 0xc000e946d0, 0xc000e5d450, 0x7173e20, 0xc000641800, 0xc000774340, 0xc000641800, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:322 +0x5b github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2(0x7173e20, 0xc000641800) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:337 +0x127 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc000e94760, 0xc000e2e500, 0xc001430f00) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67395 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d17) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000b62ba0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000b62ba0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862500, 0xc000e2e780, 0xc0048624f0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67679 [select, 42 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000dfec00, 0x0, 0x5f7c380) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:267 +0x15d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:278 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc00332d500, 0x5f7c140, 0xc00175b540, 0xc000805b50, 0xc5af1f, 0xc00120b73c) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:854 +0xf8e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc00332d500, 0xc000216428, 0x179f530) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:719 +0x62 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).withRetry(0xc00120b680, 0xc000216550, 0xc000216520, 0x0, 0xc000216540) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:573 +0x43a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc00120b680, 0x5f7c140, 0xc00175b540, 0xc0014c88f0, 0x17) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:718 +0x168 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchClient).Recv(0xc005832a10, 0x0, 0x0, 0xc004096720) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:75 +0x85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1.1(0x71cf8a0, 0xc005832a10, 0xc006bbc200, 0xc0001307e0, 0x7173e20, 0xc004096720, 0xd34f7b, 0xc004096740) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:459 +0x5a github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue.func1.1(0x7173e20, 0xc004096720) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:474 +0x7e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc005832a30, 0xc000e2e500, 0xc004096690) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67533 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc000017506) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc001a32030) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc001a32030) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f53f0, 0xc001bd2be0, 0xc0002f53e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67377 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d18) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc0034121e0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0034121e0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048622c0, 0xc000e2e780, 0xc0048622b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67461 [runnable]: sync.(*Mutex).Unlock(0xc00098dd10) /usr/local/go/src/sync/mutex.go:179 +0x82 context.(*cancelCtx).Done(0xc00098dd00, 0x0) /usr/local/go/src/context/context.go:330 +0x95 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader(0xc000a84600, 0x7fffffff, 0x1) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:268 +0xa1 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*Stream).RecvCompress(...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:278 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvAndDecompress(0xc000406f40, 0xc000a84600, 0x0, 0x0, 0x7fffffff, 0xc0019f51d0, 0x715bda0, 0xa61e5f8, 0x100010000, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:647 +0xc3 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc000406f40, 0x7f595172e140, 0xc0002b5df0, 0xc000a84600, 0x0, 0x0, 0x5f805e0, 0xc00098dcc0, 0x7fffffff, 0xc0019f51d0, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:684 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*csAttempt).recvMsg(0xc0019e4280, 0x5f805e0, 0xc00098dcc0, 0xc0019f51d0, 0xc5af1f, 0xc000f4253c) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:868 +0x2ce github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg.func1(0xc0019e4280, 0x2c, 0x2c) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:719 +0x62 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).withRetry(0xc000f42480, 0xc0019f53d0, 0xc0019f53a0, 0xc000f42530, 0xc0007b8e00) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:577 +0xe2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc000f42480, 0x5f805e0, 0xc00098dcc0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:718 +0x168 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x7173d60, 0xc001610140, 0x615060a, 0x1d, 0x5f804c0, 0xc003297080, 0x5f805e0, 0xc00098dcc0, 0xc00103aa80, 0xc003940640, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:73 +0x195 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x7173d60, 0xc001610140, 0x615060a, 0x1d, 0x5f804c0, 0xc003297080, 0x5f805e0, 0xc00098dcc0, 0xc00103aa80, 0x62a8ec0, ...) /go/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:47 +0xccb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc00103aa80, 0x7173d60, 0xc001610140, 0x615060a, 0x1d, 0x5f804c0, 0xc003297080, 0x5f805e0, 0xc00098dcc0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:35 +0x193 github.com/cockroachdb/cockroach/pkg/rpc.(*heartbeatClient).Ping(0xc0004ce120, 0x7173d60, 0xc001610140, 0xc003297080, 0x0, 0x0, 0x0, 0x6b49d27d, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/heartbeat.pb.go:196 +0x106 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3.1(...) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1090 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat.func3(0x7173d60, 0xc001610140, 0xd, 0xc0033aea01) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1097 +0xddc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc000e2e500, 0x7173d60, 0xc001610140, 0x6120f23, 0xd, 0xc0019f5d50, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:302 +0x151 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc0016b2000, 0xc00782a690, 0xc0031ec420, 0xf, 0xc0040787e0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1074 +0x624 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1.1(0x7173e20, 0xc0067f21e0) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1000 +0x95 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103cfe0, 0xc000e2e500, 0xc00782a7d0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67502 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174ff) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc001713ec0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc001713ec0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f4ee0, 0xc001bd2be0, 0xc0002f4ed0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67440 [select, 42 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0039401c0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:115 +0x203 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:106 +0x2b9 goroutine 67314 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*NodeLiveness).StartHeartbeat.func1(0x7173e20, 0xc000b638c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/node_liveness.go:500 +0x6d2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0032f2250, 0xc000e2e780, 0xc0033ae5a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67388 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d10) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc002a48150) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc002a48150) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862420, 0xc000e2e780, 0xc004862410) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 68095 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).handleMergeInProgressError(0xc000912000, 0x7173e20, 0xc001566660, 0xc0019e4480, 0xc005ce8280, 0xa61e5f8, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:409 +0x215 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc000912000, 0x7173e20, 0xc001566660, 0xc0019e4480, 0x62a1838, 0x0, 0xc005ce8280) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:284 +0x802 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithRangeID(0xc000912000, 0x7173e20, 0xc001566660, 0x2, 0xc0019e4480, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:94 +0xa8c github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc000912000, 0x7173e20, 0xc001566630, 0x6b49d27d, 0x53, 0x200000002, 0x2, 0x0, 0x2, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:36 +0xce github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc000894700, 0x7173e20, 0xc0015665a0, 0x6b49d27d, 0x53, 0x200000002, 0x2, 0x0, 0x2, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:202 +0x6c4 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc00107ac60, 0x7173e20, 0xc0015665a0, 0x0, 0x0, 0x200000002, 0x2, 0x0, 0x2, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:177 +0x206 github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.(*multiTestContextKVTransport).SendNext.func1(0x7173e20, 0xc0015665a0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_test.go:596 +0xd7 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTask(0xc001bd2be0, 0x7173e20, 0xc0015665a0, 0x6114cfa, 0x8, 0xc0009e3558, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:285 +0x151 github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.(*multiTestContextKVTransport).SendNext(0xc005cc5780, 0x7173e20, 0xc0015665a0, 0x0, 0x0, 0x200000002, 0x2, 0x0, 0x2, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_test.go:595 +0x606 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc000130b40, 0x7173e20, 0xc0015665a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1638 +0x348 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendRPC(0xc000130b40, 0x7173e20, 0xc0015665a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:456 +0x32c github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendSingleRange(0xc000130b40, 0x7173e20, 0xc0015665a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:538 +0x3e1 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc000130b40, 0x7173e20, 0xc0015665a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1407 +0x4a0 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc000130b40, 0x7173e20, 0xc0015665a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1079 +0x1d90 github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc000130b40, 0x7173e20, 0xc001566570, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:730 +0xa99 github.com/cockroachdb/cockroach/pkg/kv.(*CrossRangeTxnWrapperSender).Send(0xc0016043e0, 0x7173e20, 0xc001566570, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:223 +0xe0 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1.1(0x7173e20, 0xc001566570, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:275 +0x138 github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).sendBatch.func1(0x7173e20, 0xc001566570) /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:306 +0xc85 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc000e94720, 0xc000e2e780, 0xc001add7e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67349 [select]: github.com/cockroachdb/cockroach/pkg/gossip.newInfoStore.func1(0x7173e20, 0xc0008b7320) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/infostore.go:190 +0x224 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0018dc1f0, 0xc000e2e500, 0xc0018dc1e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67532 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174f1) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc000e67740) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e67740) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f53c0, 0xc001bd2be0, 0xc0002f53b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67578 [select, 42 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc0059cf700) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:115 +0x203 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:106 +0x2b9 goroutine 67499 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174e4) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc001713e30) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc001713e30) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f4e80, 0xc001bd2be0, 0xc0002f4df0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67421 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d37) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc004096270) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004096270) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c4c0, 0xc000e2e780, 0xc00103c4b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67630 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000836dc0, 0xc001feb350, 0x5, 0x5, 0x0, 0x0, 0xc00038cec0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:146 +0x178 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000836dc0, 0xc001feb350, 0x5, 0x5, 0xc0016737f0, 0x15ee910, 0xc00038cec0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:140 +0x31f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc000a22c90, 0xc001feb350, 0x5, 0x5, 0xc001673820, 0x1604659, 0xc00352e780) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:435 +0x73 io.ReadAtLeast(0x70f1100, 0xc000a22c90, 0xc001feb350, 0x5, 0x5, 0x5, 0x0, 0x0, 0xc00001d000) /usr/local/go/src/io/io.go:310 +0x99 io.ReadFull(...) /usr/local/go/src/io/io.go:329 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*Stream).Read(0xc00160e500, 0xc001feb350, 0x5, 0x5, 0x1, 0x2, 0x7) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:419 +0x17d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc001feb340, 0x7fffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:508 +0x87 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvAndDecompress(0xc001feb340, 0xc00160e500, 0x0, 0x0, 0x7fffffff, 0xc001673d50, 0x715bda0, 0xa61e5f8, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:639 +0x61 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc001feb340, 0x7f595172e140, 0xc0002b5df0, 0xc00160e500, 0x0, 0x0, 0x5f7c020, 0xc003929180, 0x7fffffff, 0xc001673d50, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:684 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc0019b29c0, 0x5f7c020, 0xc003929180, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:1458 +0x2b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc0018dca40, 0x7173e20, 0xc00097bef0, 0xc00001ce00) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:142 +0x85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:84 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1.1(0x71cf900, 0xc0018dca40, 0xc0001307e0, 0x7173e20, 0xc00097bef0, 0xd34f7b, 0xc00097bf10) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:342 +0x107 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1(0x7173e20, 0xc00097bef0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:378 +0x75 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0018dca60, 0xc000e2e500, 0xc00097bec0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67318 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Client).keepalive(0xc00082c000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1288 +0x1cb created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:281 +0x1f97 goroutine 67365 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d29) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000eb80f0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000eb80f0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862130, 0xc000e2e780, 0xc004862120) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67547 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc00001750f) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854b70) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854b70) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5630, 0xc001bd2be0, 0xc0002f5620) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67383 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d14) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc003412240) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc003412240) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862380, 0xc000e2e780, 0xc004862370) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67344 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d25) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc003412330) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc003412330) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862090, 0xc000e2e780, 0xc004862080) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67424 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d19) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc004096450) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004096450) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c600, 0xc000e2e780, 0xc00103c5c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67427 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).coalescedHeartbeatsLoop(0xc000798000, 0x7173e20, 0xc0034124b0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:647 +0x1d1 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c780, 0xc000e2e780, 0xc00103c740) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67519 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174fc) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc0068548a0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0068548a0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f51b0, 0xc001bd2be0, 0xc0002f51a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67398 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d27) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000eb8750) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000eb8750) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862560, 0xc000e2e780, 0xc004862550) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67880 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).heartbeatLoop(0xc0012061d0, 0x7173d60, 0xc003940240) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:257 +0x246 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000e2e780, 0x7173d60, 0xc003940240, 0xc005384690, 0x29, 0x71fbec0, 0xc003928140, 0xc000e94c60) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:323 +0x163 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:318 +0x14b goroutine 67678 [select, 42 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func5(0xc001c66000, 0xc00120b680, 0x7173e20, 0xc000641e00) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:320 +0x121 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:319 +0x11a8 goroutine 67601 [select, 22 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc0067f2420) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc000e94070, 0xc001bd2be0, 0xc006a62140) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67596 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc004096a80) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc000e94010, 0xc001bd2be0, 0xc006a62060) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67430 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func4(0x7173e20, 0xc0009afd70) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1639 +0x451 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c910, 0xc000e2e780, 0xc00782a050) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67583 [IO wait]: internal/poll.runtime_pollWait(0x7f5939038688, 0x72, 0x70f4160) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(*pollDesc).wait(0xc001697118, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0xe4 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc001697100, 0xc001112000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x253 net.(*netFD).Read(0xc001697100, 0xc001112000, 0x8000, 0x8000, 0x0, 0x0, 0xc004a8b378) /usr/local/go/src/net/fd_unix.go:202 +0x66 net.(*conn).Read(0xc003362220, 0xc001112000, 0x8000, 0x8000, 0x0, 0x10, 0xc000466690) /usr/local/go/src/net/net.go:184 +0xa1 bufio.(*Reader).Read(0xc000efd9e0, 0xc0007a02d8, 0x9, 0x9, 0x15e5b6a, 0xc000bdc000, 0xc000fa1ca1) /usr/local/go/src/bufio/bufio.go:226 +0x827 io.ReadAtLeast(0x70eb9e0, 0xc000efd9e0, 0xc0007a02d8, 0x9, 0x9, 0x9, 0x20, 0xc000fa1c60, 0x15e59ba) /usr/local/go/src/io/io.go:310 +0x99 io.ReadFull(...) /usr/local/go/src/io/io.go:329 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc0007a02d8, 0x9, 0x9, 0x70eb9e0, 0xc000efd9e0, 0x0, 0xc000000000, 0xc005b1b4a0, 0xc000466688) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xac github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc0007a02a0, 0xc003873d70, 0xc003873d00, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xfc github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000bdc000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1237 +0x252 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:286 +0x1373 goroutine 67366 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d1c) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000e66240) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e66240) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862150, 0xc000e2e780, 0xc004862140) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67495 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc00120aa20, 0x7173da0, 0xc000110010) /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:464 +0x47e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc001bd2be0, 0x7173da0, 0xc000110010, 0xc0009c28a0, 0x28, 0x0, 0x0, 0xc0002f4d70) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:323 +0x163 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:318 +0x14b goroutine 67628 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).processQueue(0xc0001307e0, 0x1, 0xc000fc93e0, 0xc00002cb40, 0x71cf8a0, 0xc00103cf00, 0x71cf801, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:485 +0x74e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).startProcessNewQueue.func2(0x7173e20, 0xc003547d70) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:636 +0x436 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0018dc930, 0xc000e2e500, 0xc00097b530) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67426 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).raftTickLoop(0xc000798000, 0x7173e20, 0xc004096060) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:591 +0x3d2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c700, 0xc000e2e780, 0xc00103c6c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67391 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d3c) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc004096000) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004096000) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862480, 0xc000e2e780, 0xc004862470) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67562 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).raftTickLoop(0xc000894700, 0x7173e20, 0xc001472e40) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:591 +0x3d2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f59a0, 0xc001bd2be0, 0xc0002f5880) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67557 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174dd) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854c90) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854c90) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5780, 0xc001bd2be0, 0xc0002f5770) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67385 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d35) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000e660c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e660c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048623c0, 0xc000e2e780, 0xc0048623b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67433 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startLeaseRenewer.func1(0x7173e20, 0xc004096480) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1688 +0x401 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103ca50, 0xc000e2e780, 0xc00103ca10) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67492 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00352e180) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:935 +0x34d created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:282 +0x180e goroutine 67416 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d39) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc0040961b0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0040961b0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c3a0, 0xc000e2e780, 0xc00103c380) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67411 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startLeaseRenewer.func1(0x7173e20, 0xc002a49410) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1688 +0x401 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862c70, 0xc001bd2be0, 0xc004862c60) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67542 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174d8) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc002a494a0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc002a494a0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5590, 0xc001bd2be0, 0xc0002f5570) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67600 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc004096b10) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc000e94060, 0xc001bd2be0, 0xc006a62120) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 68256 [chan send, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration.func4(0x7173da0, 0xc000110010, 0xc00181a100, 0xc, 0xc, 0xc00035afc0, 0xc00095f510, 0xc000869800, 0x3) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1474 +0x329 created by github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1453 +0xce6 goroutine 67428 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Start.func2(0x7173e20, 0xc003412480) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1498 +0x159 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c890, 0xc000e2e780, 0xc001dc0020) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67556 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174f2) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc001472e10) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc001472e10) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5760, 0xc001bd2be0, 0xc0002f5740) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67537 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174dc) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc0067f3170) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0067f3170) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f54b0, 0xc001bd2be0, 0xc0002f54a0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67548 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc000017500) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc000e67620) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e67620) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5650, 0xc001bd2be0, 0xc0002f5640) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67521 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc000017503) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854900) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854900) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f51f0, 0xc001bd2be0, 0xc0002f51e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67379 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d13) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc0034120c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0034120c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862300, 0xc000e2e780, 0xc0048622f0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67342 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d1e) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc003412390) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc003412390) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862050, 0xc000e2e780, 0xc004862040) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67505 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174fe) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc0067f2ed0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc0067f2ed0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f4f60, 0xc001bd2be0, 0xc0002f4f40) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 68242 [chan send, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration.func4(0x7173da0, 0xc000110010, 0xc00181a100, 0xc, 0xc, 0xc00035afc0, 0xc00095f510, 0xc000869800, 0x1) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1474 +0x329 created by github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1453 +0xce6 goroutine 67546 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174eb) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc000e67680) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e67680) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5610, 0xc001bd2be0, 0xc0002f5600) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67514 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174df) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc000e675f0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e675f0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5110, 0xc001bd2be0, 0xc0002f5100) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67496 [chan receive, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func1(0x7173e20, 0xc0040975f0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:156 +0x63 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f4d90, 0xc001bd2be0, 0xc0003a32e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67493 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).run(0xc00120a7e0, 0x7173da0, 0xc000110010) /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:464 +0x47e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc001bd2be0, 0x7173da0, 0xc000110010, 0xc0009c27b0, 0x22, 0x0, 0x0, 0xc0002f4d40) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:323 +0x163 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:318 +0x14b goroutine 972066 [runnable]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0003a5550, 0xc003e90480, 0x71d6920, 0xc00352e000, 0xc000a84800) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:715 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:715 +0xb9 goroutine 67373 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d16) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000e66270) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e66270) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862240, 0xc000e2e780, 0xc004862230) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67375 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d21) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc003412030) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc003412030) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862280, 0xc000e2e780, 0xc004862270) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67498 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174d4) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc000e5bda0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e5bda0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f4de0, 0xc001bd2be0, 0xc0002f4dd0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67567 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func4(0x7173e20, 0xc006854840) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1639 +0x451 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5a50, 0xc001bd2be0, 0xc00091bbd0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67325 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc0040969c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc003da0130, 0xc000e2e780, 0xc000330540) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67400 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d2c) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc002a481e0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc002a481e0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048625a0, 0xc000e2e780, 0xc004862590) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67574 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*StoreRebalancer).Start.func1(0x7173e20, 0xc001566ab0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_rebalancer.go:186 +0x213 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862c90, 0xc001bd2be0, 0xc000f3f800) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67459 [select]: github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip(0xc0014f8b00, 0x71cf7e0, 0xc0032f28c0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:193 +0xb6f github.com/cockroachdb/cockroach/pkg/gossip._Gossip_Gossip_Handler(0x6013e00, 0xc0014f8b00, 0x71b4840, 0xc000ce2000, 0xc0004c4868, 0xc0007c7200) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:569 +0xce github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc003e90480, 0x71d6920, 0xc00352e780, 0xc0007c7200, 0xc0008b7410, 0x99522c0, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1203 +0x1658 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc003e90480, 0x71d6920, 0xc00352e780, 0xc0007c7200, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1283 +0x12d8 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc00033ad80, 0xc003e90480, 0x71d6920, 0xc00352e780, 0xc0007c7200) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:717 +0xc9 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:715 +0xb9 goroutine 67263 [select, 42 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func5(0xc00103aa80, 0xc006ccac60, 0x7173e20, 0xc0006411a0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:320 +0x121 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:319 +0x11a8 goroutine 67702 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000a42e60, 0xc004e9b230, 0x5, 0x5, 0x0, 0x0, 0xc0003385e0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:146 +0x178 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000a42e60, 0xc004e9b230, 0x5, 0x5, 0xc000f937f0, 0x15ee910, 0xc0003385e0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:140 +0x31f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc0034127b0, 0xc004e9b230, 0x5, 0x5, 0xc000f93820, 0x1604659, 0xc00352e180) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:435 +0x73 io.ReadAtLeast(0x70f1100, 0xc0034127b0, 0xc004e9b230, 0x5, 0x5, 0x5, 0x0, 0x0, 0xc000990800) /usr/local/go/src/io/io.go:310 +0x99 io.ReadFull(...) /usr/local/go/src/io/io.go:329 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000a84700, 0xc004e9b230, 0x5, 0x5, 0x2, 0x1, 0x6) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/transport.go:419 +0x17d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc004e9b220, 0x7fffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:508 +0x87 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvAndDecompress(0xc004e9b220, 0xc000a84700, 0x0, 0x0, 0x7fffffff, 0xc000f93d50, 0x715bda0, 0xa61e5f8, 0x0, 0x0, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:639 +0x61 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc004e9b220, 0x7f595172e140, 0xc0002b5df0, 0xc000a84700, 0x0, 0x0, 0x5f7c020, 0xc0004060c0, 0x7fffffff, 0xc000f93d50, ...) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:684 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc000ce20c0, 0x5f7c020, 0xc0004060c0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:1458 +0x2b1 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*multiRaftRaftMessageBatchServer).Recv(0xc00103cd50, 0x7173e20, 0xc0035479e0, 0xc000990400) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/storage_services.pb.go:142 +0x85 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*lockedRaftMessageResponseStream).Recv(...) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:84 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1.1(0x71cf900, 0xc00103cd50, 0xc0001307e0, 0x7173e20, 0xc0035479e0, 0xd34f7b, 0xc003547a00) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:342 +0x107 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*RaftTransport).RaftMessageBatch.func1.1(0x7173e20, 0xc0035479e0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/raft_transport.go:378 +0x75 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103cd70, 0xc000e2e500, 0xc0035479b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67418 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d0e) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc004096210) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004096210) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c430, 0xc000e2e780, 0xc00103c420) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 68164 [chan send, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration.func4(0x7173da0, 0xc000110010, 0xc00181a100, 0xc, 0xc, 0xc00035afc0, 0xc00095f510, 0xc000869800, 0x7) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1474 +0x329 created by github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1453 +0xce6 goroutine 67501 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174d2) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc001713e90) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc001713e90) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f4ec0, 0xc001bd2be0, 0xc0002f4eb0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67579 [runnable]: time.stopTimer(0xc004e4ae68, 0x1c6b0d6) /usr/local/go/src/runtime/time.go:120 +0x43 time.(*Timer).Stop(...) /usr/local/go/src/time/sleep.go:77 github.com/cockroachdb/cockroach/pkg/util/timeutil.(*Timer).Reset(0xc006a631a0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/util/timeutil/timer.go:82 +0x90 github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc0016b2000, 0xc0008b26e0, 0xc0003a5980, 0xf, 0xc004a8aae0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1160 +0x34d github.com/cockroachdb/cockroach/pkg/rpc.(*Context).grpcDialNodeInternal.func1.1.1(0x7173e20, 0xc000e677a0) /go/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:1000 +0x95 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862e50, 0xc000e2e500, 0xc0008b2730) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67376 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d2f) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc003412060) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc003412060) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048622a0, 0xc000e2e780, 0xc004862290) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67565 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).startGossip.func4(0x7173e20, 0xc0015670b0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store.go:1639 +0x451 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5a30, 0xc001bd2be0, 0xc00091ba90) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67673 [select, 42 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00175af40) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:115 +0x203 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:106 +0x2b9 goroutine 67563 [select]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).coalescedHeartbeatsLoop(0xc000894700, 0x7173e20, 0xc001472e70) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_raft.go:647 +0x1d1 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f59c0, 0xc001bd2be0, 0xc0002f59b0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67399 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d2d) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000e662d0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e662d0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862580, 0xc000e2e780, 0xc004862570) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67248 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d2a) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000e66300) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e66300) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c0f0, 0xc000e2e780, 0xc00103c0e0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67597 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc0040966c0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc000e94020, 0xc001bd2be0, 0xc006a620c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67442 [chan receive, 42 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc000618580) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1107 +0x86e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:767 +0xfc goroutine 67332 [chan receive, 42 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc00091c000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1107 +0x86e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:767 +0xfc goroutine 67247 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d2e) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000b62f60) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000b62f60) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00103c0d0, 0xc000e2e780, 0xc00103c0c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67504 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc000017509) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc000e5be00) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000e5be00) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f4f30, 0xc001bd2be0, 0xc0002f4f20) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67327 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*baseQueue).processLoop.func1(0x7173e20, 0xc004096990) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/queue.go:808 +0x20a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc003da0150, 0xc000e2e780, 0xc0003305c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67483 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver/idalloc.(*Allocator).start.func1(0x7173e20, 0xc000eb9aa0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/idalloc/id_alloc.go:119 +0x58a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0032f22f0, 0xc000e2e780, 0xc0032f22d0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67560 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174d9) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854d20) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854d20) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f5810, 0xc001bd2be0, 0xc0002f5800) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67300 [select]: github.com/cockroachdb/cockroach/pkg/gossip.newInfoStore.func1(0x7173e20, 0xc0067f2cc0) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/infostore.go:190 +0x224 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f4500, 0xc000e2e500, 0xc0002f44f0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67675 [chan receive, 42 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).resetTransport(0xc00091c2c0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1107 +0x86e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:767 +0xfc goroutine 67451 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc00352e780) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:935 +0x34d created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Server /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_server.go:282 +0x180e goroutine 67644 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc0015c4b40, 0x1, 0x0, 0x0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:317 +0x1fb github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000a24ae0, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x2ad github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc00035b180) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:330 +0x12b created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:328 +0x17ba goroutine 67576 [chan receive, 42 minutes]: github.com/cockroachdb/cockroach/pkg/util/netutil.ListenAndServeGRPC.func1(0x7173e20, 0xc001472930) /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:45 +0x7a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862cf0, 0xc000e2e500, 0xc001472600) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67577 [IO wait, 42 minutes]: internal/poll.runtime_pollWait(0x7f5951787f48, 0x72, 0x72) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(*pollDesc).wait(0xc001696f98, 0x72, 0x0, 0x0, 0x6111b3c) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0xe4 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Accept(0xc001696f80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:384 +0x2cc net.(*netFD).accept(0xc001696f80, 0x7f591d62f488, 0x0, 0x0) /usr/local/go/src/net/fd_unix.go:238 +0x56 net.(*TCPListener).accept(0xc000f3f840, 0xc002ffa088, 0x0, 0x0) /usr/local/go/src/net/tcpsock_posix.go:139 +0x50 net.(*TCPListener).Accept(0xc000f3f840, 0x62a8dd0, 0xc0047e7380, 0x71d04a0, 0xc002ffa088) /usr/local/go/src/net/tcpsock.go:261 +0x50 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve(0xc0047e7380, 0x715b0e0, 0xc000f3f840, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:587 +0x316 github.com/cockroachdb/cockroach/pkg/util/netutil.ListenAndServeGRPC.func2(0x7173e20, 0xc000e5bfb0) /go/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:52 +0x54 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc004862d00, 0xc000e2e500, 0xc000f3f860) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67506 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc0000174e7) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc004097b30) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004097b30) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f4f90, 0xc001bd2be0, 0xc0002f4f70) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67703 [select, 42 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func5(0xc00103aa80, 0xc000f42c60, 0x7173e20, 0xc003547e30) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:320 +0x121 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:319 +0x11a8 goroutine 67846 [select, 42 minutes]: github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher.(*RequestBatcher).Send(0xc000306900, 0x7173da0, 0xc000110010, 0x2, 0x71f05c0, 0xc0009b9560, 0x0, 0x1, 0x5f7c020, 0xc000f3ef60) /go/src/github.com/cockroachdb/cockroach/pkg/internal/client/requestbatcher/batcher.go:248 +0x255 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).gcTxnRecord(0xc0007fc000, 0x7173da0, 0xc000110010, 0x2, 0xc00118f560, 0x18, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:700 +0x563 github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver.(*IntentResolver).cleanupFinishedTxnIntents.func2(0x7173da0, 0xc000110010) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/intentresolver/intent_resolver.go:739 +0x91 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc000e2e780, 0x7173da0, 0xc000110010, 0xc00051e8c0, 0x33, 0x0, 0x0, 0xc004097680) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:323 +0x163 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:318 +0x14b goroutine 67262 [select]: github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip(0xc000e5d450, 0x7173e20, 0xc000641050, 0xc000774340, 0x71cf780, 0xc000e946d0, 0xc000e2e500, 0xc00308c710, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:366 +0x577 github.com/cockroachdb/cockroach/pkg/gossip.(*client).startLocked.func1(0x7173e20, 0xc000641050) /go/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:125 +0x61f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc000e94640, 0xc000e2e500, 0xc001430dc0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67520 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0015c4710, 0xc000017504) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0015c4700) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc0017105a0, 0x7173e20, 0xc006854960) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc006854960) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0002f51d0, 0xc001bd2be0, 0xc0002f51c0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67392 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d36) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc004096030) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc004096030) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048624a0, 0xc000e2e780, 0xc004862490) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 67448 [IO wait]: internal/poll.runtime_pollWait(0x7f5951743d78, 0x72, 0x70f4160) /usr/local/go/src/runtime/netpoll.go:184 +0x55 internal/poll.(*pollDesc).wait(0xc002e56318, 0x72, 0x8000, 0x8000, 0xffffffffffffffff) /usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0xe4 internal/poll.(*pollDesc).waitRead(...) /usr/local/go/src/internal/poll/fd_poll_runtime.go:92 internal/poll.(*FD).Read(0xc002e56300, 0xc000d0c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /usr/local/go/src/internal/poll/fd_unix.go:169 +0x253 net.(*netFD).Read(0xc002e56300, 0xc000d0c000, 0x8000, 0x8000, 0x0, 0x0, 0xc000d372b8) /usr/local/go/src/net/fd_unix.go:202 +0x66 net.(*conn).Read(0xc0004ce128, 0xc000d0c000, 0x8000, 0x8000, 0x0, 0x10, 0xc00033a5a0) /usr/local/go/src/net/net.go:184 +0xa1 bufio.(*Reader).Read(0xc005e05080, 0xc00092a038, 0x9, 0x9, 0x15e5b6a, 0xc000940000, 0xc000505ca1) /usr/local/go/src/bufio/bufio.go:226 +0x827 io.ReadAtLeast(0x70eb9e0, 0xc005e05080, 0xc00092a038, 0x9, 0x9, 0x9, 0x20, 0xc000505c60, 0x15e59ba) /usr/local/go/src/io/io.go:310 +0x99 io.ReadFull(...) /usr/local/go/src/io/io.go:329 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc00092a038, 0x9, 0x9, 0x70eb9e0, 0xc005e05080, 0x0, 0xc000000000, 0xc0004d8910, 0xc00033a550) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0xac github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc00092a000, 0xc003f27a10, 0xc003f27a00, 0x0, 0x0) /go/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xfc github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc000940000) /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1237 +0x252 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport.newHTTP2Client /go/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/internal/transport/http2_client.go:286 +0x1373 goroutine 67386 [sync.Cond.Wait]: runtime.goparkunlock(...) /usr/local/go/src/runtime/proc.go:310 sync.runtime_notifyListWait(0xc0059ce090, 0xc000018d15) /usr/local/go/src/runtime/sema.go:510 +0xf8 sync.(*Cond).Wait(0xc0059ce080) /usr/local/go/src/sync/cond.go:56 +0x8e github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).worker(0xc00107a000, 0x7173e20, 0xc000b62990) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:197 +0xc6 github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*raftScheduler).Start.func2(0x7173e20, 0xc000b62990) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/scheduler.go:166 +0x56 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc0048623e0, 0xc000e2e780, 0xc0048623d0) /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0x1ba created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:191 +0xc4 goroutine 68295 [chan send, 42 minutes]: github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration.func4(0x7173da0, 0xc000110010, 0xc00181a100, 0xc, 0xc, 0xc00035afc0, 0xc00095f510, 0xc000869800, 0x5) /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1474 +0x329 created by github.com/cockroachdb/cockroach/pkg/kv/kvserver_test.TestStoreRangeMergeRHSLeaseExpiration /go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/client_merge_test.go:1453 +0xce6 FAIL github.com/cockroachdb/cockroach/pkg/kv/kvserver 2700.756s