storage: state.commit %d is out of range [%d, %d]

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/973210785/?project=164528&referrer=webhooks_plugin

Panic message:

stopper.go:182: string

Stacktrace (expand for inline code snippets):

/usr/local/go/src/runtime/asm_amd64.s#L572-L574 in runtime.call32
/usr/local/go/src/runtime/panic.go#L501-L503 in runtime.gopanic

cockroach/pkg/storage/raft.go

Lines 105 to 107 in f53d129

	log.ErrorfDepth(r.ctx, 1, format, v...)
	panic(fmt.Sprintf(format, v...))
	}

in pkg/storage.(*raftLogger).Panicf
https://github.com/cockroachdb/cockroach/blob/f53d12936efd36ea51eab6f191725d1dca2ceff3/vendor/go.etcd.io/etcd/raft/raft.go#L1473-L1475 in vendor/go.etcd.io/etcd/raft.(*raft).loadState
https://github.com/cockroachdb/cockroach/blob/f53d12936efd36ea51eab6f191725d1dca2ceff3/vendor/go.etcd.io/etcd/raft/raft.go#L376-L378 in vendor/go.etcd.io/etcd/raft.newRaft
https://github.com/cockroachdb/cockroach/blob/f53d12936efd36ea51eab6f191725d1dca2ceff3/vendor/go.etcd.io/etcd/raft/rawnode.go#L75-L77 in vendor/go.etcd.io/etcd/raft.NewRawNode

cockroach/pkg/storage/replica.go

Lines 622 to 624 in f53d129

	ctx := r.AnnotateCtx(context.TODO())
	raftGroup, err := raft.NewRawNode(newRaftConfig(
	raft.Storage((*replicaRaftStorage)(r)),

in pkg/storage.(*Replica).withRaftGroupLocked

cockroach/pkg/storage/replica.go

Lines 1998 to 2000 in f53d129

	if err := r.withRaftGroupLocked(true, func(raftGroup *raft.RawNode) (bool, error) {
	return true, nil

in pkg/storage.(*Replica).maybeInitializeRaftGroup

cockroach/pkg/storage/queue.go

Lines 437 to 439 in f53d129

	if bq.needsRaftInitialized {
	repl.maybeInitializeRaftGroup(ctx)
	}

in pkg/storage.(*baseQueue).maybeAddLocked

cockroach/pkg/storage/queue.go

Lines 410 to 412 in f53d129

	bq.mu.Lock()
	bq.maybeAddLocked(ctx, repl, now)
	bq.mu.Unlock()

in pkg/storage.(*baseQueue).MaybeAdd

cockroach/pkg/storage/scanner.go

Lines 234 to 236 in f53d129

	for _, q := range rs.queues {
	q.MaybeAdd(repl, rs.clock.Now())
	}

in pkg/storage.(*replicaScanner).waitAndProcess

cockroach/pkg/storage/scanner.go

Lines 282 to 284 in f53d129

	count++
	shouldStop = rs.waitAndProcess(ctx, stopper, start, repl)
	return !shouldStop

in pkg/storage.(*replicaScanner).scanLoop.func1.1

cockroach/pkg/storage/store.go

Lines 331 to 333 in f53d129

	repl.mu.RUnlock()
	if initialized && (destroyed.IsAlive() || destroyed.reason == destroyReasonRemovalPending) && !visitor(repl) {
	break

in pkg/storage.(*storeReplicaVisitor).Visit

cockroach/pkg/storage/scanner.go

Lines 280 to 282 in f53d129

	count := 0
	rs.replicas.Visit(func(repl *Replica) bool {
	count++

in pkg/storage.(*replicaScanner).scanLoop.func1

cockroach/pkg/util/stop/stopper.go

Lines 198 to 200 in f53d129

	defer s.stop.Done()
	f(ctx)
	}()

in pkg/util/stop.(*Stopper).RunWorker.func1

/usr/local/go/src/runtime/asm_amd64.s in runtime.call32 at line 573
/usr/local/go/src/runtime/panic.go in runtime.gopanic at line 502
pkg/storage/raft.go in pkg/storage.(*raftLogger).Panicf at line 106
vendor/go.etcd.io/etcd/raft/raft.go in vendor/go.etcd.io/etcd/raft.(*raft).loadState at line 1474
vendor/go.etcd.io/etcd/raft/raft.go in vendor/go.etcd.io/etcd/raft.newRaft at line 377
vendor/go.etcd.io/etcd/raft/rawnode.go in vendor/go.etcd.io/etcd/raft.NewRawNode at line 76
pkg/storage/replica.go in pkg/storage.(*Replica).withRaftGroupLocked at line 623
pkg/storage/replica.go in pkg/storage.(*Replica).maybeInitializeRaftGroup at line 1999
pkg/storage/queue.go in pkg/storage.(*baseQueue).maybeAddLocked at line 438
pkg/storage/queue.go in pkg/storage.(*baseQueue).MaybeAdd at line 411
pkg/storage/scanner.go in pkg/storage.(*replicaScanner).waitAndProcess at line 235
pkg/storage/scanner.go in pkg/storage.(*replicaScanner).scanLoop.func1.1 at line 283
pkg/storage/store.go in pkg/storage.(*storeReplicaVisitor).Visit at line 332
pkg/storage/scanner.go in pkg/storage.(*replicaScanner).scanLoop.func1 at line 281
pkg/util/stop/stopper.go in pkg/util/stop.(*Stopper).RunWorker.func1 at line 199

Tag	Value
Cockroach Release	v2.1.6
Cockroach SHA:	f53d129
Platform	linux amd64
Distribution	CCL
Environment	v2.1.6
Command	server
Go Version	go1.10.7
# of CPUs	1
# of Goroutines	168

[tbg]

  r.logger.Panicf("%x state.commit %d is out of range [%d, %d]", r.id, state.Commit, r.raftLog.committed, r.raftLog.lastIndex())

Single-CPU machine. It's also in the development environment running stock 2.1.6. Unclear what to make of this.