roachtest: tpcc/mixed-headroom/n5cpu16 failed

[cockroach-teamcity]

SHA: https://github.com/cockroachdb/cockroach/commits/77f26d185efb436aaac88243de19a27caa5da9b6

Parameters:

To repro, try:

# Don't forget to check out a clean suitable branch and experiment with the
# stress invocation until the desired results present themselves. For example,
# using stress instead of stressrace and passing the '-p' stressflag which
# controls concurrency.
./scripts/gceworker.sh start && ./scripts/gceworker.sh mosh
cd ~/go/src/github.com/cockroachdb/cockroach && \
stdbuf -oL -eL \
make stressrace TESTS=tpcc/mixed-headroom/n5cpu16 PKG=roachtest TESTTIMEOUT=5m STRESSFLAGS='-maxtime 20m -timeout 10m' 2>&1 | tee /tmp/stress.log

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=1509340&tab=artifacts#/tpcc/mixed-headroom/n5cpu16

The test failed on branch=master, cloud=gce:
test artifacts and logs in: /home/agent/work/.go/src/github.com/cockroachdb/cockroach/artifacts/20190926-1509340/tpcc/mixed-headroom/n5cpu16/run_1
	cluster.go:1764,tpcc.go:183,cluster.go:2120,errgroup.go:57: /home/agent/work/.go/src/github.com/cockroachdb/cockroach/bin/roachprod run teamcity-1569512843-81-n5cpu16:5 -- ./workload run tpcc --warehouses=909 --histograms=perf/stats.json  --ramp=5m0s --duration=2h0m0s {pgurl:1-4} returned:
		stderr:
		
		stdout:
		: signal: killed

[cockroach-teamcity]

SHA: https://github.com/cockroachdb/cockroach/commits/fb0bd2f43fb87b26752615bbdbc759502c2c8b0b

Parameters:

To repro, try:

# Don't forget to check out a clean suitable branch and experiment with the
# stress invocation until the desired results present themselves. For example,
# using stress instead of stressrace and passing the '-p' stressflag which
# controls concurrency.
./scripts/gceworker.sh start && ./scripts/gceworker.sh mosh
cd ~/go/src/github.com/cockroachdb/cockroach && \
stdbuf -oL -eL \
make stressrace TESTS=tpcc/mixed-headroom/n5cpu16 PKG=roachtest TESTTIMEOUT=5m STRESSFLAGS='-maxtime 20m -timeout 10m' 2>&1 | tee /tmp/stress.log

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=1509988&tab=artifacts#/tpcc/mixed-headroom/n5cpu16

The test failed on branch=master, cloud=gce:
test artifacts and logs in: /home/agent/work/.go/src/github.com/cockroachdb/cockroach/artifacts/20190926-1509988/tpcc/mixed-headroom/n5cpu16/run_1
	cluster.go:1764,tpcc.go:183,cluster.go:2120,errgroup.go:57: /home/agent/work/.go/src/github.com/cockroachdb/cockroach/bin/roachprod run teamcity-1569536809-83-n5cpu16:5 -- ./workload run tpcc --warehouses=909 --histograms=perf/stats.json  --ramp=5m0s --duration=2h0m0s {pgurl:1-4} returned:
		stderr:
		
		stdout:
		: signal: killed

[cockroach-teamcity]

SHA: https://github.com/cockroachdb/cockroach/commits/cf5c2bd2372e633d2f63e08e5bffca7c2a7ec59f

Parameters:

To repro, try:

# Don't forget to check out a clean suitable branch and experiment with the
# stress invocation until the desired results present themselves. For example,
# using stress instead of stressrace and passing the '-p' stressflag which
# controls concurrency.
./scripts/gceworker.sh start && ./scripts/gceworker.sh mosh
cd ~/go/src/github.com/cockroachdb/cockroach && \
stdbuf -oL -eL \
make stressrace TESTS=tpcc/mixed-headroom/n5cpu16 PKG=roachtest TESTTIMEOUT=5m STRESSFLAGS='-maxtime 20m -timeout 10m' 2>&1 | tee /tmp/stress.log

Failed test: https://teamcity.cockroachdb.com/viewLog.html?buildId=1510142&tab=artifacts#/tpcc/mixed-headroom/n5cpu16

The test failed on branch=master, cloud=gce:
test artifacts and logs in: /home/agent/work/.go/src/github.com/cockroachdb/cockroach/artifacts/20190927-1510142/tpcc/mixed-headroom/n5cpu16/run_1
	cluster.go:1764,tpcc.go:183,cluster.go:2120,errgroup.go:57: /home/agent/work/.go/src/github.com/cockroachdb/cockroach/bin/roachprod run teamcity-1569560965-81-n5cpu16:5 -- ./workload run tpcc --warehouses=909 --histograms=perf/stats.json  --ramp=5m0s --duration=2h0m0s {pgurl:1-4} returned:
		stderr:
		
		stdout:
		: signal: killed

[nvanbenschoten]

panic: runtime error: invalid memory address or nil pointer dereference [recovered]
	panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x50 pc=0x1bda042]

goroutine 190 [running]:
panic(0x3b4e000, 0x6f3fc70)
	/usr/local/go/src/runtime/panic.go:565 +0x2c5 fp=0xc007158660 sp=0xc0071585d0 pc=0x78d165
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Recover(0xc000583a70, 0x4955ae0, 0xc000515500)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:181 +0x121 fp=0xc0071586c0 sp=0xc007158660 pc=0x12d0211
runtime.call32(0x0, 0x4182548, 0xc000113660, 0x1800000018)
	/usr/local/go/src/runtime/asm_amd64.s:519 +0x3b fp=0xc0071586f0 sp=0xc0071586c0 pc=0x7bb1db
panic(0x3b4e000, 0x6f3fc70)
	/usr/local/go/src/runtime/panic.go:522 +0x1b5 fp=0xc007158780 sp=0xc0071586f0 pc=0x78d055
runtime.panicmem(...)
	/usr/local/go/src/runtime/panic.go:82
runtime.sigpanic()
	/usr/local/go/src/runtime/signal_unix.go:390 +0x411 fp=0xc0071587b0 sp=0xc007158780 pc=0x7a2cc1
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleChangeReplicasResult(0xc00738e000, 0x4955ae0, 0xc007cf53b0, 0xc008c0c700, 0xc0071589d0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_application_result.go:323 +0x1a2 fp=0xc0071588f8 sp=0xc0071587b0 pc=0x1bda042
github.com/cockroachdb/cockroach/pkg/storage.(*replicaStateMachine).handleNonTrivialReplicatedEvalResult(0xc00738e0c0, 0x4955ae0, 0xc007cf53b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_application_state_machine.go:1107 +0x5e7 fp=0xc007158fa0 sp=0xc0071588f8 pc=0x1bdfab7
github.com/cockroachdb/cockroach/pkg/storage.(*replicaStateMachine).ApplySideEffects(0xc00738e0c0, 0x4990340, 0xc000876008, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_application_state_machine.go:965 +0x824 fp=0xc007159390 sp=0xc007158fa0 pc=0x1bdf1d4
github.com/cockroachdb/cockroach/pkg/storage/apply.StateMachine.ApplySideEffects-fm(0x4990340, 0xc000876008, 0xc000876008, 0xc00738e448, 0xc00738e2d8, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/apply/task.go:55 +0x43 fp=0xc0071593d8 sp=0xc007159390 pc=0x1b42663
github.com/cockroachdb/cockroach/pkg/storage/apply.mapCheckedCmdIter(0x7f1d330bfe38, 0xc00738e2d8, 0xc0071594b0, 0x0, 0x0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/apply/cmd.go:182 +0x11b fp=0xc007159438 sp=0xc0071593d8 pc=0x1b416eb
github.com/cockroachdb/cockroach/pkg/storage/apply.(*Task).applyOneBatch(0xc007159908, 0x4955ae0, 0xc007cf53b0, 0x4990400, 0xc00738e278, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/apply/task.go:281 +0x228 fp=0xc0071594f0 sp=0xc007159438 pc=0x1b420a8
github.com/cockroachdb/cockroach/pkg/storage/apply.(*Task).ApplyCommittedEntries(0xc007159908, 0x4955ae0, 0xc007cf53b0, 0x0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/apply/task.go:247 +0xcf fp=0xc007159548 sp=0xc0071594f0 pc=0x1b41dff
github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc00738e000, 0x4955ae0, 0xc007cf53b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/replica_raft.go:760 +0xdbd fp=0xc007159c78 sp=0xc007159548 pc=0x1c09b0d
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue.func1(0x4955ae0, 0xc007cf53b0, 0xc00738e000, 0x4955ae0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3761 +0x12e fp=0xc007159d38 sp=0xc007159c78 pc=0x1c7ee0e
github.com/cockroachdb/cockroach/pkg/storage.(*Store).withReplicaForRequest(0xc00065c000, 0x4955ae0, 0xc007cf53b0, 0xc007de2a00, 0xc0061b7ec0, 0x0)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3514 +0x16b fp=0xc007159dc0 sp=0xc007159d38 pc=0x1c46fcb
github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc00065c000, 0x4955ae0, 0xc000515500, 0x117)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3750 +0x1f4 fp=0xc007159ef8 sp=0xc007159dc0 pc=0x1c47ce4
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc0006a2e00, 0x4955ae0, 0xc000515500)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:238 +0x214 fp=0xc007159f50 sp=0xc007159ef8 pc=0x1c34004
github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x4955ae0, 0xc000515500)
	/go/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:161 +0x3e fp=0xc007159f78 sp=0xc007159f50 pc=0x1c794ee
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc00029f180, 0xc000583a70, 0xc00029f170)
	/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:196 +0xfb fp=0xc007159fc8 sp=0xc007159f78 pc=0x12d23ab

Same issue as #41155. Fixed by #41171.