kvserver: v21.2.7: nil pointer in gRPC transport when sending snapshots

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/3269587070/?referrer=webhooks_plugin

Panic message:

panic.go:965: runtime error: invalid memory address or nil pointer dereference
--
runtime.errorString
panic.go:965: *withstack.withStack (top exception)

Stacktrace (expand for inline code snippets):

/usr/local/go/src/runtime/panic.go#L964-L966 in runtime.gopanic
/usr/local/go/src/runtime/panic.go#L211-L213 in runtime.panicmem
/usr/local/go/src/runtime/signal_unix.go#L733-L735 in runtime.sigpanic
https://github.com/cockroachdb/cockroach/blob/37dee546a7c52870a8dc58826f0cffe2afa8d47a/vendor/google.golang.org/grpc/internal/transport/transport.go#L308-L310 in google.golang.org/grpc/internal/transport.(*Stream).compareAndSwapState
https://github.com/cockroachdb/cockroach/blob/37dee546a7c52870a8dc58826f0cffe2afa8d47a/vendor/google.golang.org/grpc/internal/transport/http2_client.go#L927-L929 in google.golang.org/grpc/internal/transport.(*http2Client).Write
https://github.com/cockroachdb/cockroach/blob/37dee546a7c52870a8dc58826f0cffe2afa8d47a/vendor/google.golang.org/grpc/stream.go#L845-L847 in google.golang.org/grpc.(*clientStream).CloseSend.func1
https://github.com/cockroachdb/cockroach/blob/37dee546a7c52870a8dc58826f0cffe2afa8d47a/vendor/google.golang.org/grpc/stream.go#L661-L663 in google.golang.org/grpc.(*clientStream).withRetry
https://github.com/cockroachdb/cockroach/blob/37dee546a7c52870a8dc58826f0cffe2afa8d47a/vendor/google.golang.org/grpc/stream.go#L852-L854 in google.golang.org/grpc.(*clientStream).CloseSend

cockroach/pkg/kv/kvserver/raft_transport.go

Lines 690 to 692 in 37dee54

	defer func() {
	if err := stream.CloseSend(); err != nil {
	log.Warningf(ctx, "failed to close snapshot stream: %+v", err)

in pkg/kv/kvserver.(*RaftTransport).SendSnapshot.func1

cockroach/pkg/kv/kvserver/raft_transport.go

Lines 694 to 696 in 37dee54

	}()
	return sendSnapshot(
	ctx, t.st, stream, storePool, header, snap, newBatch, sent,

in pkg/kv/kvserver.(*RaftTransport).SendSnapshot

cockroach/pkg/kv/kvserver/replica_command.go

Lines 2524 to 2526 in 37dee54

	ctx, "send-snapshot", sendSnapshotTimeout, func(ctx context.Context) error {
	return r.store.cfg.Transport.SendSnapshot(
	ctx,

in pkg/kv/kvserver.(*Replica).sendSnapshot.func4

cockroach/pkg/util/contextutil/context.go

Lines 88 to 90 in 37dee54

	defer cancel()
	err := fn(ctx)
	if err != nil && errors.Is(ctx.Err(), context.DeadlineExceeded) {

in pkg/util/contextutil.RunWithTimeout

cockroach/pkg/kv/kvserver/replica_command.go

Lines 2522 to 2524 in 37dee54

	}
	err = contextutil.RunWithTimeout(
	ctx, "send-snapshot", sendSnapshotTimeout, func(ctx context.Context) error {

in pkg/kv/kvserver.(*Replica).sendSnapshot

cockroach/pkg/kv/kvserver/replica_command.go

Lines 1679 to 1681 in 37dee54

	// these, it would be susceptible to future similar issues.
	if err := r.sendSnapshot(ctx, rDesc, SnapshotRequest_INITIAL, priority); err != nil {
	return nil, err

in pkg/kv/kvserver.(*Replica).initializeRaftLearners

cockroach/pkg/kv/kvserver/replica_command.go

Lines 1048 to 1050 in 37dee54

	var err error
	desc, err = r.initializeRaftLearners(
	ctx, desc, priority, reason, details, adds, roachpb.LEARNER,

in pkg/kv/kvserver.(*Replica).changeReplicasImpl

cockroach/pkg/kv/kvserver/replicate_queue.go

Lines 1362 to 1364 in 37dee54

	// queue is active.
	if _, err := repl.changeReplicasImpl(ctx, desc, priority, reason, details, chgs); err != nil {
	return err

in pkg/kv/kvserver.(*replicateQueue).changeReplicas

cockroach/pkg/kv/kvserver/replicate_queue.go

Lines 1123 to 1125 in 37dee54

	if err := rq.changeReplicas(
	ctx,

in pkg/kv/kvserver.(*replicateQueue).considerRebalance

cockroach/pkg/kv/kvserver/replicate_queue.go

Lines 481 to 483 in 37dee54

	case AllocatorConsiderRebalance:
	return rq.considerRebalance(ctx, repl, voterReplicas, nonVoterReplicas, canTransferLeaseFrom, dryRun)
	case AllocatorFinalizeAtomicReplicationChange:

in pkg/kv/kvserver.(*replicateQueue).processOneChange

cockroach/pkg/kv/kvserver/replicate_queue.go

Lines 303 to 305 in 37dee54

	for {
	requeue, err := rq.processOneChange(ctx, repl, rq.canTransferLeaseFrom, false /* dryRun */)
	if isSnapshotError(err) {

in pkg/kv/kvserver.(*replicateQueue).process

cockroach/pkg/kv/kvserver/queue.go

Lines 958 to 960 in 37dee54

	realRepl, _ := repl.(*Replica)
	processed, err := bq.impl.process(ctx, realRepl, confReader)
	if err != nil {

in pkg/kv/kvserver.(*baseQueue).processReplica.func1

cockroach/pkg/util/contextutil/context.go

Lines 88 to 90 in 37dee54

	defer cancel()
	err := fn(ctx)
	if err != nil && errors.Is(ctx.Err(), context.DeadlineExceeded) {

in pkg/util/contextutil.RunWithTimeout

cockroach/pkg/kv/kvserver/queue.go

Lines 917 to 919 in 37dee54

	defer span.Finish()
	return contextutil.RunWithTimeout(ctx, fmt.Sprintf("%s queue process replica %d", bq.name, repl.GetRangeID()),
	bq.processTimeoutFunc(bq.store.ClusterSettings(), repl), func(ctx context.Context) error {

in pkg/kv/kvserver.(*baseQueue).processReplica

cockroach/pkg/kv/kvserver/queue.go

Lines 837 to 839 in 37dee54

	start := timeutil.Now()
	err := bq.processReplica(ctx, repl)

in pkg/kv/kvserver.(*baseQueue).processLoop.func2.1

cockroach/pkg/util/stop/stopper.go

Lines 441 to 443 in 37dee54

	f(ctx)
	}()

in pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2
/usr/local/go/src/runtime/asm_amd64.s#L1370-L1372 in runtime.goexit

/usr/local/go/src/runtime/panic.go in runtime.gopanic at line 965
/usr/local/go/src/runtime/panic.go in runtime.panicmem at line 212
/usr/local/go/src/runtime/signal_unix.go in runtime.sigpanic at line 734
vendor/google.golang.org/grpc/internal/transport/transport.go in google.golang.org/grpc/internal/transport.(*Stream).compareAndSwapState at line 309
vendor/google.golang.org/grpc/internal/transport/http2_client.go in google.golang.org/grpc/internal/transport.(*http2Client).Write at line 928
vendor/google.golang.org/grpc/stream.go in google.golang.org/grpc.(*clientStream).CloseSend.func1 at line 846
vendor/google.golang.org/grpc/stream.go in google.golang.org/grpc.(*clientStream).withRetry at line 662
vendor/google.golang.org/grpc/stream.go in google.golang.org/grpc.(*clientStream).CloseSend at line 853
pkg/kv/kvserver/raft_transport.go in pkg/kv/kvserver.(*RaftTransport).SendSnapshot.func1 at line 691
pkg/kv/kvserver/raft_transport.go in pkg/kv/kvserver.(*RaftTransport).SendSnapshot at line 695
pkg/kv/kvserver/replica_command.go in pkg/kv/kvserver.(*Replica).sendSnapshot.func4 at line 2525
pkg/util/contextutil/context.go in pkg/util/contextutil.RunWithTimeout at line 89
pkg/kv/kvserver/replica_command.go in pkg/kv/kvserver.(*Replica).sendSnapshot at line 2523
pkg/kv/kvserver/replica_command.go in pkg/kv/kvserver.(*Replica).initializeRaftLearners at line 1680
pkg/kv/kvserver/replica_command.go in pkg/kv/kvserver.(*Replica).changeReplicasImpl at line 1049
pkg/kv/kvserver/replicate_queue.go in pkg/kv/kvserver.(*replicateQueue).changeReplicas at line 1363
pkg/kv/kvserver/replicate_queue.go in pkg/kv/kvserver.(*replicateQueue).considerRebalance at line 1124
pkg/kv/kvserver/replicate_queue.go in pkg/kv/kvserver.(*replicateQueue).processOneChange at line 482
pkg/kv/kvserver/replicate_queue.go in pkg/kv/kvserver.(*replicateQueue).process at line 304
pkg/kv/kvserver/queue.go in pkg/kv/kvserver.(*baseQueue).processReplica.func1 at line 959
pkg/util/contextutil/context.go in pkg/util/contextutil.RunWithTimeout at line 89
pkg/kv/kvserver/queue.go in pkg/kv/kvserver.(*baseQueue).processReplica at line 918
pkg/kv/kvserver/queue.go in pkg/kv/kvserver.(*baseQueue).processLoop.func2.1 at line 838
pkg/util/stop/stopper.go in pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2 at line 442
/usr/local/go/src/runtime/asm_amd64.s in runtime.goexit at line 1371

Tag	Value
Cockroach Release	v21.2.7
Cockroach SHA:	37dee54
Platform	linux amd64
Distribution	CCL
Environment	v21.2.7
Command	server
Go Version	``
# of CPUs
# of Goroutines

Jira issue: CRDB-15285

[yuzefovich]

Huh, nil pointer in the gRPC transport. Probably unactionable.

[blathers-crl]

cc @cockroachdb/replication

[erikgrinaker]

FWIW, thought this might be related to gRPC issues that Andrei has been looking at over in #80878, but doesn't seem to be the case (those were caused by calls to ClientStream.Context() that were also addressed upstream in grpc/grpc-go#5323).

[pav-kv]

It looks like the attempt field is nil despite the promise in this comment:

https://github.com/cockroachdb/vendored/blob/78199fff2f3cf02d4cb350424d731d9d256a11e9/google.golang.org/grpc/stream.go#L471-L478

UPD: More likely attempt.s is nil.

[pav-kv]

This seems to be fixed in grpc/grpc-go#5323.

[pav-kv]

There are no recent re-occurrences of this panic in Sentry.

[pav-kv]

Although it doesn't look like we picked up this change. Reopening to investigate further.

[pav-kv]

Need to bump grpc from 1.46 to 1.47.

[erikgrinaker]

Reopening, since we had to roll back to 1.46.0 due to other test failures (#88745).

[erikgrinaker]

@nvanbenschoten I'm handing this over to KV, since it's a general bug in gRPC which requires an upgrade. We should try to get this in early in the 23.1 cycle, and fix the issues that motivated #88745.

[yuzefovich]

I also run into the panic in the gRPC that was fixed upstream in grpc/grpc-go#5323. Looking forward to getting the gRPC dependency bumped :)

[erikgrinaker]

I'll pick this up again.

[blathers-crl]

cc @cockroachdb/replication

[blathers-crl]

cc @cockroachdb/replication