sql/tests: TestMonotonicInserts failed under stress

[cockroach-teamcity]

SHA: https://github.com/cockroachdb/cockroach/commits/395d842feb97c5bd8cad2b32b71a5156c03061eb

Parameters:

TAGS=
GOFLAGS=

To repro, try:

# Don't forget to check out a clean suitable branch and experiment with the
# stress invocation until the desired results present themselves. For example,
# using stress instead of stressrace and passing the '-p' stressflag which
# controls concurrency.
./scripts/gceworker.sh start && ./scripts/gceworker.sh mosh
cd ~/go/src/github.com/cockroachdb/cockroach && \
stdbuf -oL -eL \
make stressrace TESTS=TestMonotonicInserts PKG=github.com/cockroachdb/cockroach/pkg/sql/tests TESTTIMEOUT=5m STRESSFLAGS='-maxtime 20m -timeout 10m' 2>&1 | tee /tmp/stress.log

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=1116509&tab=buildLog

panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x50 pc=0x1eb6bfd]

goroutine 15724 [running]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Recover(0xc0036bb4d0, 0x3087860, 0xc00145a3f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:183 +0x11f
panic(0x26b0040, 0x46e4960)
	/usr/local/go/src/runtime/panic.go:513 +0x1b9
github.com/cockroachdb/cockroach/pkg/storage.(*truncateDecision).raftSnapshotsForIndex(0xc001552190, 0x0, 0xc000d343c0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_log_queue.go:226 +0x3d
github.com/cockroachdb/cockroach/pkg/storage.(*truncateDecision).NumNewRaftSnapshots(0xc001552190, 0xc0027f6780)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_log_queue.go:254 +0x34
github.com/cockroachdb/cockroach/pkg/storage.(*truncateDecision).String(0xc001552190, 0xc0020e9e00, 0xc002a6b680)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_log_queue.go:273 +0x24a
github.com/cockroachdb/cockroach/pkg/storage.(*raftLogQueue).process(0xc0036453c0, 0x3087820, 0xc0020e9e00, 0xc002a6b680, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/raft_log_queue.go:449 +0x3d0
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processReplica(0xc00136e700, 0x3087820, 0xc0020e9e00, 0xc002a6b680, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:751 +0x345
github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1.2(0x3087860, 0xc00145a3f0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:645 +0xb8
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc0036bb4d0, 0x3087860, 0xc00145a3f0, 0xc000a81ad0, 0x2b, 0x0, 0x0, 0xc0027f6760)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:323 +0xe6
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:318 +0x134

goroutine 1 [chan receive]:
testing.(*T).Run(0xc000844d00, 0x29cc7b9, 0x14, 0x2ae7d40, 0x916c01)
	/usr/local/go/src/testing/testing.go:879 +0x383
testing.runTests.func1(0xc0001ce400)
	/usr/local/go/src/testing/testing.go:1119 +0x78
testing.tRunner(0xc0001ce400, 0xc00077dde8)
	/usr/local/go/src/testing/testing.go:827 +0xbf
testing.runTests(0xc00030df00, 0x46ff8a0, 0xf, 0xf, 0x7a04fc8)
	/usr/local/go/src/testing/testing.go:1117 +0x2aa
testing.(*M).Run(0xc00032b800, 0x0)
	/usr/local/go/src/testing/testing.go:1034 +0x165
github.com/cockroachdb/cockroach/pkg/sql/tests_test.TestMain(0xc00032b800)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/tests/main_test.go:36 +0xb5
main.main()
	_testmain.go:74 +0x13d

goroutine 6 [syscall]:
os/signal.signal_recv(0x0)
	/usr/local/go/src/runtime/sigqueue.go:139 +0x9c
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
	/usr/local/go/src/os/signal/signal_unix.go:29 +0x41

goroutine 7 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1211 +0xf2
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:627 +0x11e

goroutine 8 [chan receive]:
github.com/cockroachdb/cockroach/pkg/util/log.signalFlusher()
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:634 +0xab
created by github.com/cockroachdb/cockroach/pkg/util/log.init.0
	/go/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:628 +0x136

goroutine 10218 [select]:
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x3087860, 0xc003eed1d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:94 +0x13f
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00370d960, 0xc0036bb4d0, 0xc00370d950)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:198 +0xe1
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker

[tbg]

Fixed by #34399